This is an automated email from the ASF dual-hosted git repository.
maximebeauchemin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/superset.git
The following commit(s) were added to refs/heads/master by this push:
new b3559f644c chore: simplify Dockerfile package install calls with bash
wrappers (#31034)
b3559f644c is described below
commit b3559f644c85d55b90357af80c6a606c97bebeba
Author: Maxime Beauchemin <[email protected]>
AuthorDate: Mon Dec 2 17:57:01 2024 -0800
chore: simplify Dockerfile package install calls with bash wrappers (#31034)
---
Dockerfile | 179 ++++++++++++++++++++++--------------------
docker/apt-install.sh | 51 ++++++++++++
docker/pip-install.sh | 64 +++++++++++++++
superset-websocket/Dockerfile | 2 +-
4 files changed, 211 insertions(+), 85 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 9b48d03963..9a89cef357 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -20,44 +20,38 @@
######################################################################
ARG PY_VER=3.10-slim-bookworm
-# if BUILDPLATFORM is null, set it to 'amd64' (or leave as is otherwise).
+# If BUILDPLATFORM is null, set it to 'amd64' (or leave as is otherwise).
ARG BUILDPLATFORM=${BUILDPLATFORM:-amd64}
FROM --platform=${BUILDPLATFORM} node:20-bullseye-slim AS superset-node
+# Arguments for build configuration
ARG NPM_BUILD_CMD="build"
+ARG BUILD_TRANSLATIONS="false" # Include translations in the final build
+ARG DEV_MODE="false" # Skip frontend build in dev mode
+ARG INCLUDE_CHROMIUM="true" # Include headless Chromium for alerts & reports
+ARG INCLUDE_FIREFOX="false" # Include headless Firefox if enabled
-# Include translations in the final build. The default supports en only to
-# reduce complexity and weight for those only using en
-ARG BUILD_TRANSLATIONS="false"
-
-# Used by docker-compose to skip the frontend build,
-# in dev we mount the repo and build the frontend inside docker
-ARG DEV_MODE="false"
-
-# Include headless browsers? Allows for alerts, reports & thumbnails, but
bloats the images
-ARG INCLUDE_CHROMIUM="true"
-ARG INCLUDE_FIREFOX="false"
-
-# Somehow we need python3 + build-essential on this side of the house to
install node-gyp
-RUN apt-get update -qq \
- && apt-get install \
- -yqq --no-install-recommends \
- build-essential \
- python3 \
- zstd
+# Install system dependencies required for node-gyp
+RUN --mount=type=bind,source=./docker,target=/docker \
+ /docker/apt-install.sh build-essential python3 zstd
+# Define environment variables for frontend build
ENV BUILD_CMD=${NPM_BUILD_CMD} \
PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
-# NPM ci first, as to NOT invalidate previous steps except for when
package.json changes
-RUN
--mount=type=bind,target=/frontend-mem-nag.sh,src=./docker/frontend-mem-nag.sh \
- /frontend-mem-nag.sh
+# Run the frontend memory monitoring script
+RUN --mount=type=bind,source=./docker,target=/docker \
+ /docker/frontend-mem-nag.sh
WORKDIR /app/superset-frontend
-# Creating empty folders to avoid errors when running COPY later on
-RUN mkdir -p /app/superset/static/assets
-RUN
--mount=type=bind,target=./package.json,src=./superset-frontend/package.json \
-
--mount=type=bind,target=./package-lock.json,src=./superset-frontend/package-lock.json
\
+
+# Create necessary folders to avoid errors in subsequent steps
+RUN mkdir -p /app/superset/static/assets \
+ /app/superset/translations
+
+# Mount package files and install dependencies if not in dev mode
+RUN
--mount=type=bind,source=./superset-frontend/package.json,target=./package.json
\
+
--mount=type=bind,source=./superset-frontend/package-lock.json,target=./package-lock.json
\
if [ "$DEV_MODE" = "false" ]; then \
npm ci; \
else \
@@ -66,33 +60,39 @@ RUN
--mount=type=bind,target=./package.json,src=./superset-frontend/package.json
# Runs the webpack build process
COPY superset-frontend /app/superset-frontend
-# This copies the .po files needed for translation
-RUN mkdir -p /app/superset/translations
+
+
+# Copy translation files
COPY superset/translations /app/superset/translations
+
+# Build the frontend if not in dev mode
RUN if [ "$DEV_MODE" = "false" ]; then \
BUILD_TRANSLATIONS=$BUILD_TRANSLATIONS npm run ${BUILD_CMD}; \
else \
echo "Skipping 'npm run ${BUILD_CMD}' in dev mode"; \
fi
-
-# Compiles .json files from the .po files, then deletes the .po files
+# Compile .json files from .po translations (if required) and clean up .po
files
RUN if [ "$BUILD_TRANSLATIONS" = "true" ]; then \
npm run build-translation; \
else \
echo "Skipping translations as requested by build flag"; \
- fi
-RUN rm /app/superset/translations/*/LC_MESSAGES/*.po
-RUN rm /app/superset/translations/messages.pot
+ fi \
+ # removing translations files regardless
+ && rm -rf /app/superset/translations/*/LC_MESSAGES/*.po \
+ /app/superset/translations/messages.pot
+
+# Transition to Python base image
FROM python:${PY_VER} AS python-base
+RUN pip install --no-cache-dir --upgrade setuptools pip
+
######################################################################
# Final lean image...
######################################################################
FROM python-base AS lean
-# Include translations in the final build. The default supports en only to
-# reduce complexity and weight for those only using en
+# Build argument for including translations
ARG BUILD_TRANSLATIONS="false"
WORKDIR /app
@@ -104,9 +104,16 @@ ENV LANG=C.UTF-8 \
SUPERSET_HOME="/app/superset_home" \
SUPERSET_PORT=8088
-RUN mkdir -p ${PYTHONPATH} superset/static requirements superset-frontend
apache_superset.egg-info requirements \
+# Set up necessary directories and user
+RUN --mount=type=bind,source=./docker,target=/docker \
+ mkdir -p ${PYTHONPATH} \
+ superset/static \
+ requirements \
+ superset-frontend \
+ apache_superset.egg-info \
+ requirements \
&& useradd --user-group -d ${SUPERSET_HOME} -m --no-log-init --shell
/bin/bash superset \
- && apt-get update -qq && apt-get install -yqq --no-install-recommends \
+ && /docker/apt-install.sh \
curl \
libsasl2-dev \
libsasl2-modules-gssapi-mit \
@@ -117,58 +124,62 @@ RUN mkdir -p ${PYTHONPATH} superset/static requirements
superset-frontend apache
&& chown -R superset:superset ./* \
&& rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
+# Copy required files for Python build
COPY --chown=superset:superset pyproject.toml setup.py MANIFEST.in README.md ./
-# setup.py uses the version information in package.json
COPY --chown=superset:superset superset-frontend/package.json
superset-frontend/
COPY --chown=superset:superset requirements/base.txt requirements/
COPY --chown=superset:superset scripts/check-env.py scripts/
-RUN --mount=type=cache,target=/root/.cache/pip \
- apt-get update -qq && apt-get install -yqq --no-install-recommends \
- build-essential \
- && pip install --no-cache-dir --upgrade setuptools pip \
- && pip install --no-cache-dir -r requirements/base.txt \
- && apt-get autoremove -yqq --purge build-essential \
- && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
-# Copy the compiled frontend assets
+# Install Python dependencies using docker/pip-install.sh
+RUN --mount=type=bind,source=./docker,target=/docker \
+ --mount=type=cache,target=/root/.cache/pip \
+ /docker/pip-install.sh --requires-build-essential -r requirements/base.txt
+
+# Copy the compiled frontend assets from the node image
COPY --chown=superset:superset --from=superset-node
/app/superset/static/assets superset/static/assets
-## Lastly, let's install superset itself
+# Copy the main Superset source code
COPY --chown=superset:superset superset superset
-RUN --mount=type=cache,target=/root/.cache/pip \
- pip install --no-cache-dir -e .
-# Copy the .json translations from the frontend layer
+# Install Superset itself using docker/pip-install.sh
+RUN --mount=type=bind,source=./docker,target=/docker \
+ --mount=type=cache,target=/root/.cache/pip \
+ /docker/pip-install.sh -e .
+
+# Copy .json translations from the node image
COPY --chown=superset:superset --from=superset-node /app/superset/translations
superset/translations
-# Compile translations for the backend - this generates .mo files, then
deletes the .po files
+# Compile backend translations and clean up
COPY ./scripts/translations/generate_mo_files.sh ./scripts/translations/
RUN if [ "$BUILD_TRANSLATIONS" = "true" ]; then \
./scripts/translations/generate_mo_files.sh \
- && chown -R superset:superset superset/translations \
- && rm superset/translations/messages.pot \
- && rm superset/translations/*/LC_MESSAGES/*.po; \
- else \
- echo "Skipping translations as requested by build flag"; \
- fi
+ && chown -R superset:superset superset/translations; \
+ fi \
+ && rm -rf superset/translations/messages.pot \
+ superset/translations/*/LC_MESSAGES/*.po
+# Add server run script
COPY --chmod=755 ./docker/run-server.sh /usr/bin/
-USER superset
+# Set user and healthcheck
+USER superset
HEALTHCHECK CMD curl -f "http://localhost:${SUPERSET_PORT}/health";
+# Expose port and set CMD
EXPOSE ${SUPERSET_PORT}
-
CMD ["/usr/bin/run-server.sh"]
+
######################################################################
# Dev image...
######################################################################
FROM lean AS dev
USER root
-RUN apt-get update -qq \
- && apt-get install -yqq --no-install-recommends \
+
+# Install dev dependencies
+RUN --mount=type=bind,source=./docker,target=/docker \
+ /docker/apt-install.sh \
libnss3 \
libdbus-glib-1-2 \
libgtk-3-0 \
@@ -176,46 +187,46 @@ RUN apt-get update -qq \
libasound2 \
libxtst6 \
git \
- pkg-config \
- && rm -rf /var/cache/apt/archives/* /var/lib/apt/lists/*
+ pkg-config
+# Install Playwright and its dependencies
RUN --mount=type=cache,target=/root/.cache/pip \
- pip install --no-cache-dir playwright
-RUN playwright install-deps
+ pip install playwright \
+ && playwright install-deps
+# Optionally install Chromium
RUN if [ "$INCLUDE_CHROMIUM" = "true" ]; then \
playwright install chromium; \
else \
- echo "Skipping translations in dev mode"; \
+ echo "Skipping Chromium installation in dev mode"; \
fi
-# Install GeckoDriver WebDriver
-ARG GECKODRIVER_VERSION=v0.34.0 \
- FIREFOX_VERSION=125.0.3
-
-RUN if [ "$INCLUDE_FIREFOX" = "true" ]; then \
- apt-get update -qq \
- && apt-get install -yqq --no-install-recommends wget bzip2 \
+# Install GeckoDriver WebDriver and Firefox (if required)
+ARG GECKODRIVER_VERSION=v0.34.0
+ARG FIREFOX_VERSION=125.0.3
+RUN --mount=type=bind,source=./docker,target=/docker \
+ if [ "$INCLUDE_FIREFOX" = "true" ]; then \
+ /docker/apt-install.sh wget bzip2 \
&& wget -q
https://github.com/mozilla/geckodriver/releases/download/${GECKODRIVER_VERSION}/geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz
-O - | tar xfz - -C /usr/local/bin \
&& wget -q
https://download-installer.cdn.mozilla.net/pub/firefox/releases/${FIREFOX_VERSION}/linux-x86_64/en-US/firefox-${FIREFOX_VERSION}.tar.bz2
-O - | tar xfj - -C /opt \
&& ln -s /opt/firefox/firefox /usr/local/bin/firefox \
&& apt-get autoremove -yqq --purge wget bzip2 && rm -rf
/var/[log,tmp]/* /tmp/* /var/lib/apt/lists/* /var/cache/apt/archives/*; \
+ else \
+ echo "Skipping Firefox installation in dev mode"; \
fi
-# Installing mysql client os-level dependencies in dev image only because GPL
-RUN apt-get install -yqq --no-install-recommends \
- default-libmysqlclient-dev \
- && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
+# Install MySQL client dependencies
+RUN --mount=type=bind,source=./docker,target=/docker \
+ /docker/apt-install.sh default-libmysqlclient-dev
+# Copy development requirements and install them
COPY --chown=superset:superset requirements/development.txt requirements/
-RUN --mount=type=cache,target=/root/.cache/pip \
- apt-get update -qq && apt-get install -yqq --no-install-recommends \
- build-essential \
- && pip install --no-cache-dir -r requirements/development.txt \
- && apt-get autoremove -yqq --purge build-essential \
- && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
+RUN --mount=type=bind,source=./docker,target=/docker \
+ --mount=type=cache,target=/root/.cache/pip \
+ /docker/pip-install.sh --requires-build-essential -r
requirements/development.txt
USER superset
+
######################################################################
# CI image...
######################################################################
diff --git a/docker/apt-install.sh b/docker/apt-install.sh
new file mode 100755
index 0000000000..bd9152bebb
--- /dev/null
+++ b/docker/apt-install.sh
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+set -euo pipefail
+
+# Ensure this script is run as root
+if [[ $EUID -ne 0 ]]; then
+ echo "This script must be run as root" >&2
+ exit 1
+fi
+
+# Check for required arguments
+if [[ $# -lt 1 ]]; then
+ echo "Usage: $0 <package1> [<package2> ...]" >&2
+ exit 1
+fi
+
+# Colors for better logging (optional)
+GREEN='\033[0;32m'
+RED='\033[0;31m'
+RESET='\033[0m'
+
+# Install packages with clean-up
+echo -e "${GREEN}Updating package lists...${RESET}"
+apt-get update -qq
+
+echo -e "${GREEN}Installing packages: $@${RESET}"
+apt-get install -yqq --no-install-recommends "$@"
+
+echo -e "${GREEN}Autoremoving unnecessary packages...${RESET}"
+apt-get autoremove -y
+
+echo -e "${GREEN}Cleaning up package cache and metadata...${RESET}"
+apt-get clean
+rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* /tmp/* /var/tmp/*
+
+echo -e "${GREEN}Installation and cleanup complete.${RESET}"
diff --git a/docker/pip-install.sh b/docker/pip-install.sh
new file mode 100755
index 0000000000..2defc7d1e7
--- /dev/null
+++ b/docker/pip-install.sh
@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+set -euo pipefail
+
+# Default flag
+REQUIRES_BUILD_ESSENTIAL=false
+USE_CACHE=true
+
+# Filter arguments
+ARGS=()
+for arg in "$@"; do
+ case "$arg" in
+ --requires-build-essential)
+ REQUIRES_BUILD_ESSENTIAL=true
+ ;;
+ --no-cache)
+ USE_CACHE=false
+ ;;
+ *)
+ ARGS+=("$arg")
+ ;;
+ esac
+done
+
+# Install build-essential if required
+if $REQUIRES_BUILD_ESSENTIAL; then
+ echo "Installing build-essential for package builds..."
+ apt-get update -qq \
+ && apt-get install -yqq --no-install-recommends build-essential
+fi
+
+# Choose whether to use pip cache
+if $USE_CACHE; then
+ echo "Using pip cache..."
+ pip install "${ARGS[@]}"
+else
+ echo "Disabling pip cache..."
+ pip install --no-cache-dir "${ARGS[@]}"
+fi
+
+# Remove build-essential if it was installed
+if $REQUIRES_BUILD_ESSENTIAL; then
+ echo "Removing build-essential to keep the image lean..."
+ apt-get autoremove -yqq --purge build-essential \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
+fi
+
+echo "Python packages installed successfully."
diff --git a/superset-websocket/Dockerfile b/superset-websocket/Dockerfile
index 4cc2117f07..ac6e4a2999 100644
--- a/superset-websocket/Dockerfile
+++ b/superset-websocket/Dockerfile
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-FROM node:16-alpine as build
+FROM node:16-alpine AS build
WORKDIR /home/superset-websocket
