This is an automated email from the ASF dual-hosted git repository.
maximebeauchemin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/superset.git
The following commit(s) were added to refs/heads/master by this push:
new 1f17b975d6 fix: docker refactor (#31385)
1f17b975d6 is described below
commit 1f17b975d6845cde59d19ef4c3de044e001f19d7
Author: Maxime Beauchemin <[email protected]>
AuthorDate: Fri Dec 13 16:17:44 2024 -0800
fix: docker refactor (#31385)
---
.dockerignore | 1 +
.github/workflows/ephemeral-env.yml | 11 ++
Dockerfile | 257 ++++++++++++++++-----------------
UPDATING.md | 1 +
docker/.env | 1 +
docker/docker-bootstrap.sh | 5 +
docker/{ => entrypoints}/docker-ci.sh | 0
docker/{ => entrypoints}/run-server.sh | 0
docker/pip-install.sh | 4 +-
requirements/translations.in | 1 +
requirements/translations.txt | 9 ++
11 files changed, 156 insertions(+), 134 deletions(-)
diff --git a/.dockerignore b/.dockerignore
index e79b8f6e1e..33b76412b6 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -45,3 +45,4 @@ superset-frontend/coverage/
superset/static/assets/
superset-websocket/dist/
venv
+.venv
diff --git a/.github/workflows/ephemeral-env.yml
b/.github/workflows/ephemeral-env.yml
index 9b5539ed63..acf3b0cc72 100644
--- a/.github/workflows/ephemeral-env.yml
+++ b/.github/workflows/ephemeral-env.yml
@@ -148,9 +148,20 @@ jobs:
- name: Setup supersetbot
uses: ./.github/actions/setup-supersetbot/
+ - name: Try to login to DockerHub
+ if: steps.check.outputs.python || steps.check.outputs.frontend ||
steps.check.outputs.docker
+ continue-on-error: true
+ uses: docker/login-action@v3
+ with:
+ username: ${{ secrets.DOCKERHUB_USER }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
+
- name: Build ephemeral env image
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ with:
+ username: ${{ secrets.DOCKERHUB_USER }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
run: |
supersetbot docker \
--preset ci \
diff --git a/Dockerfile b/Dockerfile
index 60ba12eabf..4ee3093089 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -22,26 +22,29 @@ ARG PY_VER=3.10-slim-bookworm
# If BUILDPLATFORM is null, set it to 'amd64' (or leave as is otherwise).
ARG BUILDPLATFORM=${BUILDPLATFORM:-amd64}
+
+######################################################################
+# superset-node used for building frontend assets
+######################################################################
FROM --platform=${BUILDPLATFORM} node:20-bullseye-slim AS superset-node
+ARG BUILD_TRANSLATIONS="false" # Include translations in the final build
+ENV BUILD_TRANSLATIONS=${BUILD_TRANSLATIONS}
+ARG DEV_MODE="false" # Skip frontend build in dev mode
+ENV DEV_MODE=${DEV_MODE}
+COPY docker/ /app/docker/
# Arguments for build configuration
ARG NPM_BUILD_CMD="build"
-ARG BUILD_TRANSLATIONS="false" # Include translations in the final build
-ARG DEV_MODE="false" # Skip frontend build in dev mode
-ARG INCLUDE_CHROMIUM="true" # Include headless Chromium for alerts & reports
-ARG INCLUDE_FIREFOX="false" # Include headless Firefox if enabled
# Install system dependencies required for node-gyp
-RUN --mount=type=bind,source=./docker,target=/docker \
- /docker/apt-install.sh build-essential python3 zstd
+RUN /app/docker/apt-install.sh build-essential python3 zstd
# Define environment variables for frontend build
ENV BUILD_CMD=${NPM_BUILD_CMD} \
PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
# Run the frontend memory monitoring script
-RUN --mount=type=bind,source=./docker,target=/docker \
- /docker/frontend-mem-nag.sh
+RUN /app/docker/frontend-mem-nag.sh
WORKDIR /app/superset-frontend
@@ -49,6 +52,9 @@ WORKDIR /app/superset-frontend
RUN mkdir -p /app/superset/static/assets \
/app/superset/translations
+# Copy translation files
+COPY superset/translations /app/superset/translations
+
# Mount package files and install dependencies if not in dev mode
RUN
--mount=type=bind,source=./superset-frontend/package.json,target=./package.json
\
--mount=type=bind,source=./superset-frontend/package-lock.json,target=./package-lock.json
\
@@ -61,41 +67,28 @@ RUN
--mount=type=bind,source=./superset-frontend/package.json,target=./package.j
# Runs the webpack build process
COPY superset-frontend /app/superset-frontend
-
-# Copy translation files
-COPY superset/translations /app/superset/translations
-
# Build the frontend if not in dev mode
RUN if [ "$DEV_MODE" = "false" ]; then \
- BUILD_TRANSLATIONS=$BUILD_TRANSLATIONS npm run ${BUILD_CMD}; \
+ echo "Running 'npm run ${BUILD_CMD}'"; \
+ if [ "$BUILD_TRANSLATIONS" = "true" ]; then \
+ npm run build-translation; \
+ fi; \
+ npm run ${BUILD_CMD}; \
else \
echo "Skipping 'npm run ${BUILD_CMD}' in dev mode"; \
- fi
-
-# Compile .json files from .po translations (if required) and clean up .po
files
-RUN if [ "$BUILD_TRANSLATIONS" = "true" ]; then \
- npm run build-translation; \
- else \
- echo "Skipping translations as requested by build flag"; \
- fi \
- # removing translations files regardless
- && rm -rf /app/superset/translations/*/LC_MESSAGES/*.po \
- /app/superset/translations/messages.pot
-
+ fi && \
+ rm -rf /app/superset/translations/*/*/*.po
-# Transition to Python base image
-FROM python:${PY_VER} AS python-base
-RUN pip install --no-cache-dir --upgrade setuptools pip uv
######################################################################
-# Final lean image...
+# Base python layer
######################################################################
-FROM python-base AS lean
-
-# Build argument for including translations
-ARG BUILD_TRANSLATIONS="false"
+FROM python:${PY_VER} AS python-base
+ARG BUILD_TRANSLATIONS="false" # Include translations in the final build
+ENV BUILD_TRANSLATIONS=${BUILD_TRANSLATIONS}
+ARG DEV_MODE="false" # Skip frontend build in dev mode
+ENV DEV_MODE=${DEV_MODE}
-WORKDIR /app
ENV LANG=C.UTF-8 \
LC_ALL=C.UTF-8 \
SUPERSET_ENV=production \
@@ -104,126 +97,128 @@ ENV LANG=C.UTF-8 \
SUPERSET_HOME="/app/superset_home" \
SUPERSET_PORT=8088
+
+RUN useradd --user-group -d ${SUPERSET_HOME} -m --no-log-init --shell
/bin/bash superset
+
+# Some bash scripts needed throughout the layers
+COPY --chmod=755 docker/*.sh /app/docker/
+
+RUN pip install --no-cache-dir --upgrade setuptools pip uv
+
+# Using uv as it's faster/simpler than pip
+RUN uv venv /app/.venv
+ENV PATH="/app/.venv/bin:${PATH}"
+
+# Install Playwright and optionally setup headless browsers
+ARG INCLUDE_CHROMIUM="true"
+ARG INCLUDE_FIREFOX="false"
+RUN --mount=type=cache,target=/root/.cache/pip \
+ if [ "$INCLUDE_CHROMIUM" = "true" ] || [ "$INCLUDE_FIREFOX" = "true" ];
then \
+ pip install playwright && \
+ playwright install-deps && \
+ if [ "$INCLUDE_CHROMIUM" = "true" ]; then playwright install chromium;
fi && \
+ if [ "$INCLUDE_FIREFOX" = "true" ]; then playwright install firefox;
fi; \
+ else \
+ echo "Skipping browser installation"; \
+ fi
+
+######################################################################
+# Python translation compiler layer
+######################################################################
+FROM python-base AS python-translation-compiler
+
+# Install Python dependencies using docker/pip-install.sh
+COPY requirements/translations.txt requirements/
+RUN --mount=type=cache,target=/root/.cache/pip \
+ /app/docker/pip-install.sh -r requirements/translations.txt
+
+COPY superset/translations/ /app/translations_mo/
+RUN pybabel compile -d /app/translations_mo | true && \
+ rm -f /app/translations_mo/*/*/*.po
+
+######################################################################
+# Python APP common layer
+######################################################################
+FROM python-base AS python-common
+# Copy the entrypoints, make them executable in userspace
+COPY --chmod=755 docker/entrypoints /app/docker/entrypoints
+
+WORKDIR /app
# Set up necessary directories and user
-RUN --mount=type=bind,source=./docker,target=/docker \
- mkdir -p ${PYTHONPATH} \
+RUN mkdir -p \
+ ${SUPERSET_HOME} \
+ ${PYTHONPATH} \
superset/static \
requirements \
superset-frontend \
apache_superset.egg-info \
requirements \
- && useradd --user-group -d ${SUPERSET_HOME} -m --no-log-init --shell
/bin/bash superset \
- && /docker/apt-install.sh \
- curl \
- libsasl2-dev \
- libsasl2-modules-gssapi-mit \
- libpq-dev \
- libecpg-dev \
- libldap2-dev \
- && touch superset/static/version_info.json \
- && chown -R superset:superset ./* \
- && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
+ && touch superset/static/version_info.json
# Copy required files for Python build
-COPY --chown=superset:superset pyproject.toml setup.py MANIFEST.in README.md ./
-COPY --chown=superset:superset superset-frontend/package.json
superset-frontend/
-COPY --chown=superset:superset requirements/base.txt requirements/
-COPY --chown=superset:superset scripts/check-env.py scripts/
-
-# Install Python dependencies using docker/pip-install.sh
-RUN --mount=type=bind,source=./docker,target=/docker \
- --mount=type=cache,target=/root/.cache/pip \
- /docker/pip-install.sh --requires-build-essential -r requirements/base.txt
-
-# Copy the compiled frontend assets from the node image
-COPY --chown=superset:superset --from=superset-node
/app/superset/static/assets superset/static/assets
+COPY pyproject.toml setup.py MANIFEST.in README.md ./
+COPY superset-frontend/package.json superset-frontend/
+COPY scripts/check-env.py scripts/
-# Copy the main Superset source code
-COPY --chown=superset:superset superset superset
+# keeping for backward compatibility
+COPY --chmod=755 ./docker/entrypoints/run-server.sh /usr/bin/
-# Install Superset itself using docker/pip-install.sh
-RUN --mount=type=bind,source=./docker,target=/docker \
- --mount=type=cache,target=/root/.cache/pip \
- /docker/pip-install.sh -e .
+# Some debian libs
+RUN /app/docker/apt-install.sh \
+ curl \
+ libsasl2-dev \
+ libsasl2-modules-gssapi-mit \
+ libpq-dev \
+ libecpg-dev \
+ libldap2-dev
-# Copy .json translations from the node image
-COPY --chown=superset:superset --from=superset-node /app/superset/translations
superset/translations
+# Copy compiled things from previous stages
+COPY --from=superset-node /app/superset/static/assets superset/static/assets
-# Compile backend translations and clean up
-COPY ./scripts/translations/generate_mo_files.sh ./scripts/translations/
-RUN if [ "$BUILD_TRANSLATIONS" = "true" ]; then \
- ./scripts/translations/generate_mo_files.sh \
- && chown -R superset:superset superset/translations; \
- fi \
- && rm -rf superset/translations/messages.pot \
- superset/translations/*/LC_MESSAGES/*.po
+# Merging translations from backend and frontend stages
+COPY --from=superset-node /app/superset/translations superset/translations
+COPY --from=python-translation-compiler /app/translations_mo
superset/translations
-# Add server run script
-COPY --chmod=755 ./docker/run-server.sh /usr/bin/
-
-# Set user and healthcheck
-USER superset
HEALTHCHECK CMD curl -f "http://localhost:${SUPERSET_PORT}/health";
-
-# Expose port and set CMD
+CMD ["/app/docker/entrypoints/run-server.sh"]
EXPOSE ${SUPERSET_PORT}
-CMD ["/usr/bin/run-server.sh"]
-
######################################################################
-# Dev image...
+# Final lean image...
######################################################################
-FROM lean AS dev
-
-USER root
+FROM python-common AS lean
+COPY superset superset
-# Install dev dependencies
-RUN --mount=type=bind,source=./docker,target=/docker \
- /docker/apt-install.sh \
- libnss3 \
- libdbus-glib-1-2 \
- libgtk-3-0 \
- libx11-xcb1 \
- libasound2 \
- libxtst6 \
- git \
- pkg-config
-
-# Install Playwright and its dependencies
+# Install Python dependencies using docker/pip-install.sh
+COPY requirements/base.txt requirements/
RUN --mount=type=cache,target=/root/.cache/pip \
- uv pip install --system playwright \
- && playwright install-deps
+ /app/docker/pip-install.sh --requires-build-essential -r
requirements/base.txt && \
+ uv pip install .
-# Optionally install Chromium
-RUN if [ "$INCLUDE_CHROMIUM" = "true" ]; then \
- playwright install chromium; \
- else \
- echo "Skipping Chromium installation in dev mode"; \
- fi
+RUN python -m compileall /app/superset
-# Install GeckoDriver WebDriver and Firefox (if required)
-ARG GECKODRIVER_VERSION=v0.34.0
-ARG FIREFOX_VERSION=125.0.3
-RUN --mount=type=bind,source=./docker,target=/docker \
- if [ "$INCLUDE_FIREFOX" = "true" ]; then \
- /docker/apt-install.sh wget bzip2 \
- && wget -q
https://github.com/mozilla/geckodriver/releases/download/${GECKODRIVER_VERSION}/geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz
-O - | tar xfz - -C /usr/local/bin \
- && wget -q
https://download-installer.cdn.mozilla.net/pub/firefox/releases/${FIREFOX_VERSION}/linux-x86_64/en-US/firefox-${FIREFOX_VERSION}.tar.bz2
-O - | tar xfj - -C /opt \
- && ln -s /opt/firefox/firefox /usr/local/bin/firefox \
- && apt-get autoremove -yqq --purge wget bzip2 && rm -rf
/var/[log,tmp]/* /tmp/* /var/lib/apt/lists/* /var/cache/apt/archives/*; \
- else \
- echo "Skipping Firefox installation in dev mode"; \
- fi
+USER superset
+
+######################################################################
+# Dev image...
+######################################################################
+FROM python-common AS dev
+COPY superset superset
-# Install MySQL client dependencies
-RUN --mount=type=bind,source=./docker,target=/docker \
- /docker/apt-install.sh default-libmysqlclient-dev
+# Debian libs needed for dev
+RUN /app/docker/apt-install.sh \
+ git \
+ pkg-config \
+ default-libmysqlclient-dev
# Copy development requirements and install them
-COPY --chown=superset:superset requirements/development.txt requirements/
-RUN --mount=type=bind,source=./docker,target=/docker \
- --mount=type=cache,target=/root/.cache/pip \
- /docker/pip-install.sh --requires-build-essential -r
requirements/development.txt
+COPY requirements/*.txt requirements/
+# Install Python dependencies using docker/pip-install.sh
+RUN --mount=type=cache,target=/root/.cache/pip \
+ /app/docker/pip-install.sh --requires-build-essential -r
requirements/development.txt && \
+ uv pip install .
+
+RUN python -m compileall /app/superset
USER superset
@@ -232,6 +227,4 @@ USER superset
######################################################################
FROM lean AS ci
-COPY --chown=superset:superset --chmod=755 ./docker/*.sh /app/docker/
-
-CMD ["/app/docker/docker-ci.sh"]
+CMD ["/app/docker/entrypoints/docker-ci.sh"]
diff --git a/UPDATING.md b/UPDATING.md
index a5ae173667..4267ae340a 100644
--- a/UPDATING.md
+++ b/UPDATING.md
@@ -30,6 +30,7 @@ assists people when migrating to a new version.
- [30099](https://github.com/apache/superset/pull/30099) Translations are no
longer included in the default docker image builds. If your environment
requires translations, you'll want to set the docker build arg
`BUILD_TRANSACTION=true`.
- [31262](https://github.com/apache/superset/pull/31262) NOTE: deprecated
`pylint` in favor of `ruff` as our only python linter. Only affect development
workflows positively (not the release itself). It should cover most important
rules, be much faster, but some things linting rules that were enforced before
may not be enforce in the exact same way as before.
- [31173](https://github.com/apache/superset/pull/31173) Modified
`fetch_csrf_token` to align with HTTP standards, particularly regarding how
cookies are handled. If you encounter any issues related to CSRF functionality,
please report them as a new issue and reference this PR for context.
+- [31385](https://github.com/apache/superset/pull/31385) Significant docker
refactor, reducing access levels for the `superset` user, streamlining layer
building, ...
### Potential Downtime
diff --git a/docker/.env b/docker/.env
index 57575da76e..7511766569 100644
--- a/docker/.env
+++ b/docker/.env
@@ -17,6 +17,7 @@
COMPOSE_PROJECT_NAME=superset
+DEV_MODE=true
# database configurations (do not modify)
DATABASE_DB=superset
diff --git a/docker/docker-bootstrap.sh b/docker/docker-bootstrap.sh
index 2f0b29ce34..1a4e04be94 100755
--- a/docker/docker-bootstrap.sh
+++ b/docker/docker-bootstrap.sh
@@ -18,6 +18,11 @@
set -eo pipefail
+# Make python interactive
+if [ "$DEV_MODE" == "true" ]; then
+ echo "Reinstalling the app in editable mode"
+ uv pip install -e .
+fi
REQUIREMENTS_LOCAL="/app/docker/requirements-local.txt"
# If Cypress run – overwrite the password for admin and export env variables
if [ "$CYPRESS_CONFIG" == "true" ]; then
diff --git a/docker/docker-ci.sh b/docker/entrypoints/docker-ci.sh
similarity index 100%
rename from docker/docker-ci.sh
rename to docker/entrypoints/docker-ci.sh
diff --git a/docker/run-server.sh b/docker/entrypoints/run-server.sh
similarity index 100%
rename from docker/run-server.sh
rename to docker/entrypoints/run-server.sh
diff --git a/docker/pip-install.sh b/docker/pip-install.sh
index 7e69a6efba..7deb4fa19a 100755
--- a/docker/pip-install.sh
+++ b/docker/pip-install.sh
@@ -47,10 +47,10 @@ fi
# Choose whether to use pip cache
if $USE_CACHE; then
echo "Using pip cache..."
- uv pip install --system "${ARGS[@]}"
+ uv pip install "${ARGS[@]}"
else
echo "Disabling pip cache..."
- uv pip install --system --no-cache-dir "${ARGS[@]}"
+ uv pip install --no-cache-dir "${ARGS[@]}"
fi
# Remove build-essential if it was installed
diff --git a/requirements/translations.in b/requirements/translations.in
new file mode 100644
index 0000000000..98f65931c4
--- /dev/null
+++ b/requirements/translations.in
@@ -0,0 +1 @@
+babel
diff --git a/requirements/translations.txt b/requirements/translations.txt
new file mode 100644
index 0000000000..4eab2d21f4
--- /dev/null
+++ b/requirements/translations.txt
@@ -0,0 +1,9 @@
+# SHA1:cad160f3d4cd7c33896f42a479eeaa1b5bedc5fb
+#
+# This file is autogenerated by pip-compile-multi
+# To update, run:
+#
+# pip-compile-multi
+#
+babel==2.16.0
+ # via -r requirements/translations.in
