This is an automated email from the ASF dual-hosted git repository.
jiadongb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/texera.git
The following commit(s) were added to refs/heads/main by this push:
new 84976e9439 feat(dockerfile): add build argument to toggle R support in
computing unit images (#3929)
84976e9439 is described below
commit 84976e9439439c1c8f8db57caee04ad8688b020e
Author: Jiadong Bai <[email protected]>
AuthorDate: Sat Oct 18 16:43:11 2025 -0700
feat(dockerfile): add build argument to toggle R support in computing unit
images (#3929)
## Summary
- Add `WITH_R_SUPPORT` build argument to both
`computing-unit-master.dockerfile` and
`computing-unit-worker.dockerfile`
- Conditionally install R dependencies based on the build flag
- Defaults to `false` to reduce image size and build time when R support
is not needed
## Related Issue
Closes #3928
## Changes
- Added `ARG WITH_R_SUPPORT=false` (default: false for backward
compatibility)
- Conditionally install R-related system packages (gfortran,
libreadline, libxml2, etc.)
- Conditionally compile and install R 4.3.3
- Conditionally install R packages (coro, dplyr, arrow)
## Impact
**Without R support (default):**
- Significantly smaller image size (~500MB+ reduction)
- Faster build time (avoids 10+ minute R compilation)
- Suitable for deployments that don't use R operators
**With R support:**
```bash
docker build --build-arg WITH_R_SUPPORT=true -f
bin/computing-unit-master.dockerfile .
```
---
bin/computing-unit-master.dockerfile | 70 +++++++++++++++++++++---------------
bin/computing-unit-worker.dockerfile | 67 +++++++++++++++++++++++++++++-----
2 files changed, 100 insertions(+), 37 deletions(-)
diff --git a/bin/computing-unit-master.dockerfile
b/bin/computing-unit-master.dockerfile
index e94c4158ff..35bd130cb2 100644
--- a/bin/computing-unit-master.dockerfile
+++ b/bin/computing-unit-master.dockerfile
@@ -43,19 +43,24 @@ RUN unzip amber/target/universal/amber-*.zip -d
amber/target/
FROM eclipse-temurin:11-jdk-jammy AS runtime
+# Build argument to enable/disable R support (default: true for backward
compatibility)
+ARG WITH_R_SUPPORT=false
+
WORKDIR /texera/amber
COPY --from=build /texera/amber/r-requirements.txt /tmp/r-requirements.txt
COPY --from=build /texera/amber/requirements.txt /tmp/requirements.txt
COPY --from=build /texera/amber/operator-requirements.txt
/tmp/operator-requirements.txt
-# Install Python & R runtime dependencies
+# Install Python runtime dependencies (always) and R runtime dependencies
(conditional)
RUN apt-get update && apt-get install -y \
python3-pip \
python3-dev \
libpq-dev \
- gfortran \
curl \
+ unzip \
+ $(if [ "$WITH_R_SUPPORT" = "true" ]; then echo "\
+ gfortran \
build-essential \
libreadline-dev \
libncurses-dev \
@@ -66,36 +71,43 @@ RUN apt-get update && apt-get install -y \
liblzma-dev \
libpcre++-dev \
libpango1.0-dev \
- libcurl4-openssl-dev \
- unzip \
+ libcurl4-openssl-dev"; fi) \
&& apt-get clean
-# Install R and needed libraries
+# Install R and needed libraries (conditional)
ENV R_VERSION=4.3.3
-RUN curl -O https://cran.r-project.org/src/base/R-4/R-${R_VERSION}.tar.gz && \
- tar -xf R-${R_VERSION}.tar.gz && \
- cd R-${R_VERSION} && \
- ./configure --prefix=/usr/local \
- --enable-R-shlib \
- --with-blas \
- --with-lapack && \
- make -j 4 && \
- make install && \
- cd .. && \
- rm -rf R-${R_VERSION}* && R --version && pip3 install --upgrade pip
setuptools wheel && \
- pip3 install -r /tmp/requirements.txt && \
- pip3 install -r /tmp/operator-requirements.txt && \
- pip3 install -r /tmp/r-requirements.txt
-# Install R packages, pinning arrow to 14.0.2.1 explicitly
-RUN Rscript -e "options(repos = c(CRAN = 'https://cran.r-project.org')); \
- install.packages(c('coro', 'dplyr'), \
- Ncpus = parallel::detectCores())" && \
- Rscript -e "options(repos = c(CRAN = 'https://cran.r-project.org')); \
- if (!requireNamespace('remotes', quietly=TRUE)) \
- install.packages('remotes'); \
- remotes::install_version('arrow', version='14.0.2.1', \
- repos='https://cran.r-project.org', upgrade='never'); \
- cat('R arrow version: ',
as.character(packageVersion('arrow')), '\n')"
+RUN if [ "$WITH_R_SUPPORT" = "true" ]; then \
+ curl -O https://cran.r-project.org/src/base/R-4/R-${R_VERSION}.tar.gz
&& \
+ tar -xf R-${R_VERSION}.tar.gz && \
+ cd R-${R_VERSION} && \
+ ./configure --prefix=/usr/local \
+ --enable-R-shlib \
+ --with-blas \
+ --with-lapack && \
+ make -j 4 && \
+ make install && \
+ cd .. && \
+ rm -rf R-${R_VERSION}* && R --version && pip3 install --upgrade pip
setuptools wheel && \
+ pip3 install -r /tmp/requirements.txt && \
+ pip3 install -r /tmp/operator-requirements.txt && \
+ pip3 install -r /tmp/r-requirements.txt; \
+ else \
+ pip3 install --upgrade pip setuptools wheel && \
+ pip3 install -r /tmp/requirements.txt && \
+ pip3 install -r /tmp/operator-requirements.txt; \
+ fi
+# Install R packages, pinning arrow to 14.0.2.1 explicitly (conditional)
+RUN if [ "$WITH_R_SUPPORT" = "true" ]; then \
+ Rscript -e "options(repos = c(CRAN = 'https://cran.r-project.org')); \
+ install.packages(c('coro', 'dplyr'), \
+ Ncpus = parallel::detectCores())" && \
+ Rscript -e "options(repos = c(CRAN = 'https://cran.r-project.org')); \
+ if (!requireNamespace('remotes', quietly=TRUE)) \
+ install.packages('remotes'); \
+ remotes::install_version('arrow', version='14.0.2.1', \
+ repos='https://cran.r-project.org', upgrade='never'); \
+ cat('R arrow version: ',
as.character(packageVersion('arrow')), '\n')"; \
+ fi
ENV LD_LIBRARY_PATH=/usr/local/lib/R/lib:$LD_LIBRARY_PATH
# Copy the built texera binary from the build phase
diff --git a/bin/computing-unit-worker.dockerfile
b/bin/computing-unit-worker.dockerfile
index 9fd3867170..fbf9d6b77e 100644
--- a/bin/computing-unit-worker.dockerfile
+++ b/bin/computing-unit-worker.dockerfile
@@ -43,25 +43,76 @@ RUN unzip amber/target/universal/amber-*.zip -d
amber/target/
FROM eclipse-temurin:11-jre-jammy AS runtime
+# Build argument to enable/disable R support (default: false for backward
compatibility)
+ARG WITH_R_SUPPORT=false
+
WORKDIR /texera/amber
+COPY --from=build /texera/amber/r-requirements.txt /tmp/r-requirements.txt
COPY --from=build /texera/amber/requirements.txt /tmp/requirements.txt
COPY --from=build /texera/amber/operator-requirements.txt
/tmp/operator-requirements.txt
-# Install Python runtime and dependencies
+# Install Python runtime dependencies (always) and R runtime dependencies
(conditional)
RUN apt-get update && apt-get install -y \
python3-pip \
python3-dev \
libpq-dev \
+ $(if [ "$WITH_R_SUPPORT" = "true" ]; then echo "\
+ gfortran \
+ curl \
+ build-essential \
+ libreadline-dev \
+ libncurses-dev \
+ libssl-dev \
+ libxml2-dev \
+ xorg-dev \
+ libbz2-dev \
+ liblzma-dev \
+ libpcre++-dev \
+ libpango1.0-dev \
+ libcurl4-openssl-dev"; fi) \
&& apt-get clean
-RUN pip3 install --upgrade pip setuptools wheel
-RUN pip3 install python-lsp-server python-lsp-server[websockets]
-
-# Install requirements with a fallback for wordcloud
-RUN pip3 install -r /tmp/requirements.txt
-RUN pip3 install --no-cache-dir --find-links https://pypi.org/simple/ -r
/tmp/operator-requirements.txt || \
- pip3 install --no-cache-dir wordcloud==1.9.2
+# Install R and needed libraries (conditional)
+ENV R_VERSION=4.3.3
+RUN if [ "$WITH_R_SUPPORT" = "true" ]; then \
+ curl -O https://cran.r-project.org/src/base/R-4/R-${R_VERSION}.tar.gz
&& \
+ tar -xf R-${R_VERSION}.tar.gz && \
+ cd R-${R_VERSION} && \
+ ./configure --prefix=/usr/local \
+ --enable-R-shlib \
+ --with-blas \
+ --with-lapack && \
+ make -j 4 && \
+ make install && \
+ cd .. && \
+ rm -rf R-${R_VERSION}* && R --version && pip3 install --upgrade pip
setuptools wheel && \
+ pip3 install python-lsp-server python-lsp-server[websockets] && \
+ pip3 install -r /tmp/requirements.txt && \
+ pip3 install --no-cache-dir --find-links https://pypi.org/simple/ -r
/tmp/operator-requirements.txt || \
+ pip3 install --no-cache-dir wordcloud==1.9.2 && \
+ pip3 install -r /tmp/r-requirements.txt; \
+ else \
+ pip3 install --upgrade pip setuptools wheel && \
+ pip3 install python-lsp-server python-lsp-server[websockets] && \
+ pip3 install -r /tmp/requirements.txt && \
+ pip3 install --no-cache-dir --find-links https://pypi.org/simple/ -r
/tmp/operator-requirements.txt || \
+ pip3 install --no-cache-dir wordcloud==1.9.2; \
+ fi
+
+# Install R packages, pinning arrow to 14.0.2.1 explicitly (conditional)
+RUN if [ "$WITH_R_SUPPORT" = "true" ]; then \
+ Rscript -e "options(repos = c(CRAN = 'https://cran.r-project.org')); \
+ install.packages(c('coro', 'dplyr'), \
+ Ncpus = parallel::detectCores())" && \
+ Rscript -e "options(repos = c(CRAN = 'https://cran.r-project.org')); \
+ if (!requireNamespace('remotes', quietly=TRUE)) \
+ install.packages('remotes'); \
+ remotes::install_version('arrow', version='14.0.2.1', \
+ repos='https://cran.r-project.org', upgrade='never'); \
+ cat('R arrow version: ',
as.character(packageVersion('arrow')), '\n')"; \
+ fi
+ENV LD_LIBRARY_PATH=/usr/local/lib/R/lib:$LD_LIBRARY_PATH
# Copy the built texera binary from the build phase
COPY --from=build /texera/amber/target/amber-* /texera/amber/
