This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 40c97296c33 branch-3.0: [feature](doris compose) run docker as host
user (#43091)
40c97296c33 is described below
commit 40c97296c33c80b3546367e0d804af15045f4735
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Thu Nov 7 14:13:00 2024 +0800
branch-3.0: [feature](doris compose) run docker as host user (#43091)
PR Body: Our workmates run doris compose on the same dev machine.
Sometimes some docker doris_be consume high cpu. We want to find out it
belongs to who. But if show processes with command 'ps -elf' on host
machine, we only find out that its owner is root.
What a pitty. So we let doris compose docker run as host user. Then use
command 'ps -elf' can known who it belongs to.
To achive this goal, we pass host user and uid into docker container,
and let container add a new user the same with it, then run command with
this new account.
If user still want to run as root in container, they need to pass
options '--root' when creating a new doris compose cluster.
```
# run as root in container
python doris-compose.py up --root ...
#run as host user in container
python doris-compose.py up ...
```
Cherry-picked from #43040
Co-authored-by: yujun <[email protected]>
---
docker/runtime/doris-compose/Dockerfile | 6 +-
docker/runtime/doris-compose/cluster.py | 92 +++++++++++-----------
docker/runtime/doris-compose/command.py | 9 ++-
.../runtime/doris-compose/resource/entrypoint.sh | 68 ++++++++++++++++
4 files changed, 124 insertions(+), 51 deletions(-)
diff --git a/docker/runtime/doris-compose/Dockerfile
b/docker/runtime/doris-compose/Dockerfile
index 48d94d612df..501574e372e 100644
--- a/docker/runtime/doris-compose/Dockerfile
+++ b/docker/runtime/doris-compose/Dockerfile
@@ -38,7 +38,7 @@ RUN sed -i s@/deb.debian.org/@/mirrors.aliyun.com/@g
/etc/apt/sources.list
RUN apt-get clean
RUN apt-get update && \
- apt-get install -y default-mysql-client python lsof tzdata curl unzip
patchelf jq procps util-linux && \
+ apt-get install -y default-mysql-client python lsof tzdata curl unzip
patchelf jq procps util-linux gosu && \
ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \
dpkg-reconfigure -f noninteractive tzdata && \
apt-get clean
@@ -48,14 +48,14 @@ RUN curl -f
https://repo1.maven.org/maven2/org/jacoco/jacoco/${JACOCO_VERSION}/j
unzip jacoco.zip -d /jacoco
# cloud
-COPY cloud/CMakeLists.txt cloud/output* output/ms* /opt/apache-doris/cloud/
+COPY --chmod=777 README.md cloud/output* output/ms* /opt/apache-doris/cloud/
RUN mkdir /opt/apache-doris/fdb
RUN if [ -d /opt/apache-doris/cloud/bin ]; then \
sed -i 's/\<chmod\>/echo/g' /opt/apache-doris/cloud/bin/start.sh ; \
fi
# fe and be
-COPY output /opt/apache-doris/
+COPY --chmod=777 output /opt/apache-doris/
# in docker, run 'chmod 755 doris_be' first time cost 1min, remove it.
RUN sed -i 's/\<chmod\>/echo/g' /opt/apache-doris/be/bin/start_be.sh
diff --git a/docker/runtime/doris-compose/cluster.py
b/docker/runtime/doris-compose/cluster.py
index 985ef27113b..abd59bcd0e9 100644
--- a/docker/runtime/doris-compose/cluster.py
+++ b/docker/runtime/doris-compose/cluster.py
@@ -292,32 +292,27 @@ class Node(object):
enable_coverage = self.cluster.coverage_dir
envs = {
- "MY_IP":
- self.get_ip(),
- "MY_ID":
- self.id,
- "MY_TYPE":
- self.node_type(),
- "FE_QUERY_PORT":
- FE_QUERY_PORT,
- "FE_EDITLOG_PORT":
- FE_EDITLOG_PORT,
- "BE_HEARTBEAT_PORT":
- BE_HEARTBEAT_PORT,
- "DORIS_HOME":
- os.path.join(self.docker_home_dir()),
- "STOP_GRACE":
- 1 if enable_coverage else 0,
- "IS_CLOUD":
- 1 if self.cluster.is_cloud else 0,
- "SQL_MODE_NODE_MGR":
- 1 if hasattr(self.cluster, 'sql_mode_node_mgr')
- and self.cluster.sql_mode_node_mgr else 0
+ "MY_IP": self.get_ip(),
+ "MY_ID": self.id,
+ "MY_TYPE": self.node_type(),
+ "FE_QUERY_PORT": FE_QUERY_PORT,
+ "FE_EDITLOG_PORT": FE_EDITLOG_PORT,
+ "BE_HEARTBEAT_PORT": BE_HEARTBEAT_PORT,
+ "DORIS_HOME": os.path.join(self.docker_home_dir()),
+ "STOP_GRACE": 1 if enable_coverage else 0,
+ "IS_CLOUD": 1 if self.cluster.is_cloud else 0,
+ "SQL_MODE_NODE_MGR": 1 if self.cluster.sql_mode_node_mgr else 0,
}
if self.cluster.is_cloud:
envs["META_SERVICE_ENDPOINT"] = self.cluster.get_meta_server_addr()
+ # run as host user
+ if not getattr(self.cluster, 'is_root_user', True):
+ envs["HOST_USER"] = getpass.getuser()
+ envs["HOST_UID"] = os.getuid()
+ envs["HOST_GID"] = os.getgid()
+
if enable_coverage:
outfile = "{}/coverage/{}-coverage-{}-{}".format(
DOCKER_DORIS_PATH, self.node_type(), self.cluster.name,
@@ -332,6 +327,15 @@ class Node(object):
return envs
+ def entrypoint(self):
+ if self.start_script():
+ return [
+ "bash",
+ os.path.join(DOCKER_RESOURCE_PATH, "entrypoint.sh")
+ ] + self.start_script()
+ else:
+ return None
+
def get_add_init_config(self):
return []
@@ -441,8 +445,8 @@ class FE(Node):
def cloud_unique_id(self):
return "sql_server_{}".format(self.id)
- def entrypoint(self):
- return ["bash", os.path.join(DOCKER_RESOURCE_PATH, "init_fe.sh")]
+ def start_script(self):
+ return ["init_fe.sh"]
def docker_ports(self):
return [FE_HTTP_PORT, FE_EDITLOG_PORT, FE_RPC_PORT, FE_QUERY_PORT]
@@ -539,8 +543,8 @@ class BE(Node):
storage_root_path = ";".join(dir_descs) if dir_descs else '""'
f.write("\nstorage_root_path = {}\n".format(storage_root_path))
- def entrypoint(self):
- return ["bash", os.path.join(DOCKER_RESOURCE_PATH, "init_be.sh")]
+ def start_script(self):
+ return ["init_be.sh"]
def docker_env(self):
envs = super().docker_env()
@@ -591,12 +595,8 @@ class MS(CLOUD):
cfg += self.cluster.ms_config
return cfg
- def entrypoint(self):
- return [
- "bash",
- os.path.join(DOCKER_RESOURCE_PATH, "init_cloud.sh"),
- "--meta-service"
- ]
+ def start_script(self):
+ return ["init_cloud.sh", "--meta-service"]
def node_type(self):
return Node.TYPE_MS
@@ -616,11 +616,8 @@ class RECYCLE(CLOUD):
cfg += self.cluster.recycle_config
return cfg
- def entrypoint(self):
- return [
- "bash",
- os.path.join(DOCKER_RESOURCE_PATH, "init_cloud.sh"), "--recycler"
- ]
+ def start_script(self):
+ return ["init_cloud.sh", "--recycler"]
def node_type(self):
return Node.TYPE_RECYCLE
@@ -641,8 +638,8 @@ class FDB(Node):
with open(os.path.join(local_conf_dir, "fdb.cluster"), "w") as f:
f.write(self.cluster.get_fdb_cluster())
- def entrypoint(self):
- return ["bash", os.path.join(DOCKER_RESOURCE_PATH, "init_fdb.sh")]
+ def start_script(self):
+ return ["init_fdb.sh"]
def docker_home_dir(self):
return os.path.join(DOCKER_DORIS_PATH, "fdb")
@@ -659,14 +656,15 @@ class FDB(Node):
class Cluster(object):
- def __init__(self, name, subnet, image, is_cloud, fe_config, be_config,
- ms_config, recycle_config, fe_follower, be_disks, be_cluster,
- reg_be, coverage_dir, cloud_store_config, sql_mode_node_mgr,
- be_metaservice_endpoint, be_cluster_id):
+ def __init__(self, name, subnet, image, is_cloud, is_root_user, fe_config,
+ be_config, ms_config, recycle_config, fe_follower, be_disks,
+ be_cluster, reg_be, coverage_dir, cloud_store_config,
+ sql_mode_node_mgr, be_metaservice_endpoint, be_cluster_id):
self.name = name
self.subnet = subnet
self.image = image
self.is_cloud = is_cloud
+ self.is_root_user = is_root_user
self.fe_config = fe_config
self.be_config = be_config
self.ms_config = ms_config
@@ -686,9 +684,9 @@ class Cluster(object):
self.be_cluster_id = be_cluster_id
@staticmethod
- def new(name, image, is_cloud, fe_config, be_config, ms_config,
- recycle_config, fe_follower, be_disks, be_cluster, reg_be,
- coverage_dir, cloud_store_config, sql_mode_node_mgr,
+ def new(name, image, is_cloud, is_root_user, fe_config, be_config,
+ ms_config, recycle_config, fe_follower, be_disks, be_cluster,
+ reg_be, coverage_dir, cloud_store_config, sql_mode_node_mgr,
be_metaservice_endpoint, be_cluster_id):
if not os.path.exists(LOCAL_DORIS_PATH):
os.makedirs(LOCAL_DORIS_PATH, exist_ok=True)
@@ -698,8 +696,8 @@ class Cluster(object):
if os.getuid() == utils.get_path_uid(lock_file):
os.chmod(lock_file, 0o666)
subnet = gen_subnet_prefix16()
- cluster = Cluster(name, subnet, image, is_cloud, fe_config,
- be_config, ms_config, recycle_config,
+ cluster = Cluster(name, subnet, image, is_cloud, is_root_user,
+ fe_config, be_config, ms_config, recycle_config,
fe_follower, be_disks, be_cluster, reg_be,
coverage_dir, cloud_store_config,
sql_mode_node_mgr, be_metaservice_endpoint,
diff --git a/docker/runtime/doris-compose/command.py
b/docker/runtime/doris-compose/command.py
index 267c0de3ab0..99849590435 100644
--- a/docker/runtime/doris-compose/command.py
+++ b/docker/runtime/doris-compose/command.py
@@ -261,6 +261,13 @@ class UpCommand(Command):
help=
"Create cloud cluster, default is false. Only use when creating
new cluster."
)
+ parser.add_argument(
+ "--root",
+ default=False,
+ action=self._get_parser_bool_action(True),
+ help=
+ "Run cluster as root user, default is false, it will run as host
user."
+ )
parser.add_argument(
"--wait-timeout",
@@ -500,7 +507,7 @@ class UpCommand(Command):
args.add_recycle_num = 0
cluster = CLUSTER.Cluster.new(
- args.NAME, args.IMAGE, args.cloud, args.fe_config,
+ args.NAME, args.IMAGE, args.cloud, args.root, args.fe_config,
args.be_config, args.ms_config, args.recycle_config,
args.fe_follower, args.be_disks, args.be_cluster, args.reg_be,
args.coverage_dir, cloud_store_config, args.sql_mode_node_mgr,
diff --git a/docker/runtime/doris-compose/resource/entrypoint.sh
b/docker/runtime/doris-compose/resource/entrypoint.sh
new file mode 100644
index 00000000000..a3cdaaae8f1
--- /dev/null
+++ b/docker/runtime/doris-compose/resource/entrypoint.sh
@@ -0,0 +1,68 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+DIR=$(
+ cd $(dirname $0)
+ pwd
+)
+
+source $DIR/common.sh
+
+RUN_USER=root
+
+create_host_user() {
+ if [ -z ${HOST_USER} ]; then
+ health_log "no specific run user, run as root"
+ return
+ fi
+ id ${HOST_USER}
+ if [ $? -eq 0 ]; then
+ health_log "contain user ${HOST_USER}, no create new user"
+ RUN_USER=${HOST_USER}
+ return
+ fi
+ id ${HOST_UID}
+ if [ $? -eq 0 ]; then
+ health_log "contain uid ${HOST_UID}, no create new user"
+ return
+ fi
+ addgroup --gid ${HOST_GID} ${HOST_USER}
+ if [ $? -eq 0 ]; then
+ health_log "create group ${HOST_USER} with gid ${HOST_GID} succ"
+ else
+ health_log "create group ${HOST_USER} with gid ${HOST_GID} failed"
+ return
+ fi
+ adduser --disabled-password --shell /bin/bash --gecos "" --uid ${HOST_UID}
--gid ${HOST_GID} ${HOST_USER}
+ if [ $? -eq 0 ]; then
+ health_log "create user ${HOST_USER} with uid ${HOST_UID} succ"
+ RUN_USER=${HOST_USER}
+ else
+ health_log "create user ${HOST_USER} with uid ${HOST_UID} failed"
+ fi
+}
+
+create_host_user
+
+if command -v gosu 2>&1 >/dev/null; then
+ if [ -f ${LOG_FILE} ]; then
+ chown ${RUN_USER}:${RUN_USER} ${LOG_FILE}
+ fi
+ gosu ${RUN_USER} bash ${DIR}/${1} ${@:2}
+else
+ bash ${DIR}/${1} ${@:2}
+fi
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
