This is an automated email from the ASF dual-hosted git repository.
eolivelli pushed a commit to branch 2.7.2_ds_rootless
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 376867a1f9bd729566065e4ca13b1983c426e6f8
Author: Michael Marshall <47911938+michaeljmarsh...@users.noreply.github.com>
AuthorDate: Mon Feb 15 21:17:52 2021 -0700
[Issue 8751] Update Dockerfile for Pulsar and Dashboard to Create and Use
pulsar User (nonroot user) (#8796)
Fixes #8751
Pulsar does not need to run as the root user. This PR updates the pulsar
and the pulsar dashboard images to make them run as a new `pulsar` user (user
~1000~ 10000 and group 10001). This change increases the security of pulsar
images.
Update two `Dockerfile`s to create a pulsar user, chown the appropriate
directories, and then use that user by default.
- [ ] Make sure that the change passes the CI checks.
I manually verified that the docker images run with the correct user and
file permissions. As this is my first commit, I'm not familiar with pulsar
testing. Are there tests that run against the produced docker images? If so,
then there is likely no further testing needed.
(cherry picked from commit 4264a67a84d9a9f4a49cebc591c46b252dcf4e45)
---
docker/pulsar-all/Dockerfile | 12 ++++++++
docker/pulsar-standalone/Dockerfile | 3 ++
docker/pulsar/Dockerfile | 33 +++++++++++++++++-----
site2/docs/getting-started-docker.md | 5 ++++
.../docker-images/latest-version-image/Dockerfile | 6 ++++
.../latest-version-image/conf/bookie.conf | 1 +
.../latest-version-image/conf/broker.conf | 1 +
.../conf/functions_worker.conf | 1 +
.../latest-version-image/conf/global-zk.conf | 1 +
.../latest-version-image/conf/local-zk.conf | 1 +
.../latest-version-image/conf/presto_worker.conf | 3 +-
.../latest-version-image/conf/proxy.conf | 1 +
12 files changed, 60 insertions(+), 8 deletions(-)
diff --git a/docker/pulsar-all/Dockerfile b/docker/pulsar-all/Dockerfile
index 5daa406..8a5451d 100644
--- a/docker/pulsar-all/Dockerfile
+++ b/docker/pulsar-all/Dockerfile
@@ -25,8 +25,20 @@ ARG PULSAR_OFFLOADER_TARBALL
ADD ${PULSAR_IO_DIR} /connectors
ADD ${PULSAR_OFFLOADER_TARBALL} /
RUN mv /apache-pulsar-offloaders-*/offloaders /offloaders
+RUN chmod -R g=u /connectors /offloaders
FROM apachepulsar/pulsar:latest
+
+# Need permission to create directories and update file permissions
+USER root
+
+RUN mkdir /pulsar/connectors /pulsar/offloaders && \
+ chown pulsar:root /pulsar/connectors /pulsar/offloaders && \
+ chmod g=u /pulsar/connectors /pulsar/offloaders
+
+# Return to pulsar (non root) user
+USER pulsar
+
COPY --from=pulsar-all /connectors/pulsar-io-elastic-search-*.nar
/pulsar/connectors/
COPY --from=pulsar-all /connectors/pulsar-io-kinesis-*.nar /pulsar/connectors/
COPY --from=pulsar-all /connectors/pulsar-io-kafka-*.nar /pulsar/connectors/
diff --git a/docker/pulsar-standalone/Dockerfile
b/docker/pulsar-standalone/Dockerfile
index 777541b..c7fdad9 100644
--- a/docker/pulsar-standalone/Dockerfile
+++ b/docker/pulsar-standalone/Dockerfile
@@ -26,6 +26,9 @@ FROM apachepulsar/pulsar-dashboard:latest as dashboard
# Restart from
FROM openjdk:11-jdk
+# Help to make these directories persist between container restarts
+VOLUME ["/pulsar/conf", "/pulsar/data"]
+
# Note that the libpq-dev package is needed here in order to install
# the required python psycopg2 package (for postgresql) later
RUN apt-get update \
diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile
index a224c33..0dc6598 100644
--- a/docker/pulsar/Dockerfile
+++ b/docker/pulsar/Dockerfile
@@ -17,7 +17,8 @@
# under the License.
#
-# First create a stage with just the Pulsar tarball and scripts
+# First create a stage with the Pulsar tarball, the scripts, the python client,
+# the cpp client, and the data directory. Then ensure correct file permissions.
FROM busybox as pulsar
ARG PULSAR_TARBALL
@@ -34,12 +35,25 @@ COPY scripts/watch-znode.py /pulsar/bin
COPY scripts/set_python_version.sh /pulsar/bin
COPY scripts/install-pulsar-client-37.sh /pulsar/bin
+COPY target/python-client/ /pulsar/pulsar-client
+COPY target/cpp-client/ /pulsar/cpp-client
+
+RUN mkdir /pulsar/data
+
+# In order to support running this docker image as a container on OpenShift
+# the final image needs to give the root group enough permission.
+# The file permissions are maintained when copied into the target image.
+RUN chmod -R g=u /pulsar
### Create 2nd stage from OpenJDK image
### and add Python dependencies (for Pulsar functions)
FROM openjdk:11-jdk-slim
+# Create the pulsar group and user to make docker container run as a non root
user by default
+RUN groupadd -g 10001 pulsar
+RUN adduser -u 10000 --gid 10001 --disabled-login --disabled-password --gecos
'' pulsar
+
# Install some utilities
RUN apt-get update \
&& apt-get install -y netcat dnsutils less procps iputils-ping \
@@ -54,21 +68,26 @@ RUN python3.7 get-pip.py
RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10
-ADD target/python-client/ /pulsar/pulsar-client
-ADD target/cpp-client/ /pulsar/cpp-client
+# The pulsar directory is staged correctly in the first stage, above.
+# The chown and chmod ensure proper permissions for running as a non root user
and non root group
+# as well as running on OpenShift with a random user that is part of the root
group
+RUN mkdir /pulsar && chown pulsar:0 /pulsar && chmod g=u /pulsar
+COPY --from=pulsar --chown=pulsar:0 /pulsar /pulsar
+
RUN echo networkaddress.cache.ttl=1 >> $JAVA_HOME/conf/security/java.security
+
RUN apt-get update \
&& apt install -y /pulsar/cpp-client/*.deb \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
-VOLUME ["/pulsar/conf", "/pulsar/data"]
-
ENV PULSAR_ROOT_LOGGER=INFO,CONSOLE
-
-COPY --from=pulsar /pulsar /pulsar
WORKDIR /pulsar
+# This script is intentionally run as the root user to make the dependencies
+# available to the root user and the pulsar user
RUN /pulsar/bin/install-pulsar-client-37.sh
+# Switch to the pulsar user to ensure container defaults to run as a non root
user
+USER pulsar
diff --git a/site2/docs/getting-started-docker.md
b/site2/docs/getting-started-docker.md
index 56cff28..f8f6a38 100644
--- a/site2/docs/getting-started-docker.md
+++ b/site2/docs/getting-started-docker.md
@@ -27,6 +27,11 @@ and follow the instructions for your OS.
A few things to note about this command:
* The data, metadata, and configuration are persisted on Docker volumes in
order to not start "fresh" every
time the container is restarted. For details on the volumes you can use
`docker volume inspect <sourcename>`
+ * The pulsar docker image runs as user 10000, by default. In order for the
pulsar process to
+ access the mounted volumes, the host volumes (`pulsardata` and `pulsarconf`
in this example)
+ must give user 10000 read and write permissions. Alternatively, you can
specify the user to run with and then make sure
+ that the host volume gives that user read and write permissions. Otherwise,
the process will fail due to insufficient
+ permissions.
* For Docker on Windows make sure to configure it to use Linux containers
If you start Pulsar successfully, you will see `INFO`-level log messages like
this:
diff --git a/tests/docker-images/latest-version-image/Dockerfile
b/tests/docker-images/latest-version-image/Dockerfile
index eaa279a..5793722 100644
--- a/tests/docker-images/latest-version-image/Dockerfile
+++ b/tests/docker-images/latest-version-image/Dockerfile
@@ -19,6 +19,12 @@
FROM apachepulsar/pulsar-all:latest
+# Switch to run as the root user to simplify building container and then
running
+# supervisord. Each of the pulsar components are spawned by supervisord and
their
+# process configuration files specify that the process will be run as the
"pulsar" user
+# However, any processes exec'ing into the containers will run as root, by
default.
+USER root
+
RUN rm -rf /var/lib/apt/lists/* && apt update
RUN apt-get clean && apt-get update && apt-get install -y supervisor vim
procps curl git
diff --git a/tests/docker-images/latest-version-image/conf/bookie.conf
b/tests/docker-images/latest-version-image/conf/bookie.conf
index a71cb5c..f95e231 100644
--- a/tests/docker-images/latest-version-image/conf/bookie.conf
+++ b/tests/docker-images/latest-version-image/conf/bookie.conf
@@ -24,3 +24,4 @@ stdout_logfile=/var/log/pulsar/bookie.log
directory=/pulsar
environment=PULSAR_MEM="-Xmx128M
-XX:MaxDirectMemorySize=512M",PULSAR_GC="-XX:+UseG1GC",dbStorage_writeCacheMaxSizeMb="16",dbStorage_readAheadCacheMaxSizeMb="16"
command=/pulsar/bin/pulsar bookie
+user=pulsar
diff --git a/tests/docker-images/latest-version-image/conf/broker.conf
b/tests/docker-images/latest-version-image/conf/broker.conf
index f8bf8e7..bfcf601 100644
--- a/tests/docker-images/latest-version-image/conf/broker.conf
+++ b/tests/docker-images/latest-version-image/conf/broker.conf
@@ -24,4 +24,5 @@ stdout_logfile=/var/log/pulsar/broker.log
directory=/pulsar
environment=PULSAR_MEM="-Xmx128M",PULSAR_GC="-XX:+UseG1GC"
command=/pulsar/bin/pulsar broker
+user=pulsar
diff --git
a/tests/docker-images/latest-version-image/conf/functions_worker.conf
b/tests/docker-images/latest-version-image/conf/functions_worker.conf
index 3610b03..a023c1e 100644
--- a/tests/docker-images/latest-version-image/conf/functions_worker.conf
+++ b/tests/docker-images/latest-version-image/conf/functions_worker.conf
@@ -24,4 +24,5 @@ stdout_logfile=/var/log/pulsar/functions_worker.log
directory=/pulsar
environment=PULSAR_MEM="-Xmx128M",PULSAR_GC="-XX:+UseG1GC"
command=/pulsar/bin/pulsar functions-worker
+user=pulsar
diff --git a/tests/docker-images/latest-version-image/conf/global-zk.conf
b/tests/docker-images/latest-version-image/conf/global-zk.conf
index bf56c5b..6503f5f 100644
--- a/tests/docker-images/latest-version-image/conf/global-zk.conf
+++ b/tests/docker-images/latest-version-image/conf/global-zk.conf
@@ -24,4 +24,5 @@ stdout_logfile=/var/log/pulsar/global-zk.log
directory=/pulsar
environment=PULSAR_MEM="-Xmx128M",PULSAR_GC="-XX:+UseG1GC"
command=/pulsar/bin/pulsar configuration-store
+user=pulsar
diff --git a/tests/docker-images/latest-version-image/conf/local-zk.conf
b/tests/docker-images/latest-version-image/conf/local-zk.conf
index 5768193..1c98a6b 100644
--- a/tests/docker-images/latest-version-image/conf/local-zk.conf
+++ b/tests/docker-images/latest-version-image/conf/local-zk.conf
@@ -24,4 +24,5 @@ stdout_logfile=/var/log/pulsar/local-zk.log
directory=/pulsar
environment=PULSAR_MEM="-Xmx128M",PULSAR_GC="-XX:+UseG1GC"
command=/pulsar/bin/pulsar zookeeper
+user=pulsar
diff --git a/tests/docker-images/latest-version-image/conf/presto_worker.conf
b/tests/docker-images/latest-version-image/conf/presto_worker.conf
index 28e3c36..6846ca2 100644
--- a/tests/docker-images/latest-version-image/conf/presto_worker.conf
+++ b/tests/docker-images/latest-version-image/conf/presto_worker.conf
@@ -23,4 +23,5 @@ redirect_stderr=true
stdout_logfile=/var/log/pulsar/presto_worker.log
directory=/pulsar
environment=PULSAR_MEM="-Xmx128M",PULSAR_GC="-XX:+UseG1GC"
-command=/pulsar/bin/pulsar sql-worker start
\ No newline at end of file
+command=/pulsar/bin/pulsar sql-worker start
+user=pulsar
\ No newline at end of file
diff --git a/tests/docker-images/latest-version-image/conf/proxy.conf
b/tests/docker-images/latest-version-image/conf/proxy.conf
index 8bc1a53..1bed5a1 100644
--- a/tests/docker-images/latest-version-image/conf/proxy.conf
+++ b/tests/docker-images/latest-version-image/conf/proxy.conf
@@ -24,4 +24,5 @@ stdout_logfile=/var/log/pulsar/proxy.log
directory=/pulsar
environment=PULSAR_MEM="-Xmx128M",PULSAR_GC="-XX:+UseG1GC"
command=/pulsar/bin/pulsar proxy
+user=pulsar
