This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new bb2c737fece branch-3.0: [feat](doris compose) Add extra hosts option
for up command #51098 (#51183)
bb2c737fece is described below
commit bb2c737feceef8193b11f92cff3a9c8c774aa059
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Fri May 23 20:17:06 2025 +0800
branch-3.0: [feat](doris compose) Add extra hosts option for up command
#51098 (#51183)
Cherry-picked from #51098
Co-authored-by: yujun <[email protected]>
---
docker/runtime/doris-compose/Dockerfile | 9 +++++-
docker/runtime/doris-compose/Readme.md | 31 ++++++++++++++++++++
docker/runtime/doris-compose/cluster.py | 33 +++++++++++++---------
docker/runtime/doris-compose/command.py | 12 ++++++--
.../doris/regression/suite/SuiteCluster.groovy | 17 ++++++++---
5 files changed, 82 insertions(+), 20 deletions(-)
diff --git a/docker/runtime/doris-compose/Dockerfile
b/docker/runtime/doris-compose/Dockerfile
index 04b86a30ed1..7dc7221d6e3 100644
--- a/docker/runtime/doris-compose/Dockerfile
+++ b/docker/runtime/doris-compose/Dockerfile
@@ -22,7 +22,14 @@
# docker build -f docker/runtime/doris-compose/Dockerfile -t
<your-image-name>:<version> .
# choose a base image
+# doris 2.1, 3.0+, master use JDK 17
ARG JDK_IMAGE=openjdk:17-jdk-slim
+
+# doris 2.0 use JDK 8
+# build 2.0 image, example:
+# docker build --build-arg JDK_IMAGE=openjdk:8u342-jdk \
+# -f docker/runtime/doris-compose/Dockerfile \
+# -t <your-image-name>:<version> .
#ARG JDK_IMAGE=openjdk:8u342-jdk
# user can download a doris release package, extract it, then build its image
used arg `OUTPUT_PATH`
@@ -48,7 +55,7 @@ FROM ${JDK_IMAGE}
# set environment variables
ENV JACOCO_VERSION=0.8.8
-RUN sed -i s@/deb.debian.org/@/mirrors.aliyun.com/@g /etc/apt/sources.list \
+RUN sed -i -e s@/deb.debian.org/@/mirrors.aliyun.com/@g -e
s@/security.debian.org/@/mirrors.aliyun.com/@g /etc/apt/sources.list \
&& apt-get clean \
&& apt-get update \
&& apt-get install -y --no-install-recommends \
diff --git a/docker/runtime/doris-compose/Readme.md
b/docker/runtime/doris-compose/Readme.md
index 98b703be59e..34bab457872 100644
--- a/docker/runtime/doris-compose/Readme.md
+++ b/docker/runtime/doris-compose/Readme.md
@@ -49,8 +49,39 @@ If build doris use `sh build.sh --fe --be --cloud` **without
do any change on th
docker build -f docker/runtime/doris-compose/Dockerfile -t <image> .
```
+The Dockerfile default use JDK 17, for doris 2.1, 3.0, master, they all
default use JDK 17.
+
+But doris 2.0 still use JDK 8, for build 2.0 image, user need specific use JDK
8 with arg `JDK_IMAGE=openjdk:8u342-jdk`. Here is build 2.0 image command:
+
+
+```shell
+docker build -f docker/runtime/doris-compose/Dockerfile \
+ --build-arg JDK_IMAGE=openjdk:8u342-jdk \
+ -t <image> .
+```
+
+
The `<image>` is the name you want the docker image to have.
+User can also download a doris release package from [Doris
Home](https://doris.apache.org/docs/releasenotes/all-release) or [Doris
Github](https://github.com/apache/doris/releases), extract it, then build its
image with arg `OUTPUT_PATH`
+
+for example:
+
+ ```shell
+cd ~/tmp
+wget
https://apache-doris-releases.oss-accelerate.aliyuncs.com/apache-doris-3.0.5-bin-x64.tar.gz
+tar xvf apache-doris-3.0.5-bin-x64.tar.gz # after extract, there will be a
directory ./apache-doris-3.0.5-bin-x64/{fe, be, ms}
+
+# -f: the Dockerfile file
+# -t: the builded image
+# . : current directory, here it's ~/tmp, then output path is
~/tmp/apache-doris-3.0.5-bin-x64
+docker build \
+ --build-arg OUTPUT_PATH=./apache-doris-3.0.5-bin-x64 \
+ -f ~/workspace/doris/docker/runtime/doris-compose/Dockerfile \
+ -t my-doris:v3.0.5 \
+ .
+```
+
### 3. Install the dependent python library in
'docker/runtime/doris-compose/requirements.txt'
`PyYAML` of certain version not always fit other libraries' requirements. So
we suggest to use a individual environment using `venv` or `conda`.
diff --git a/docker/runtime/doris-compose/cluster.py
b/docker/runtime/doris-compose/cluster.py
index 552b9c038c9..87313546c69 100644
--- a/docker/runtime/doris-compose/cluster.py
+++ b/docker/runtime/doris-compose/cluster.py
@@ -145,8 +145,7 @@ def gen_subnet_prefix16():
def get_master_fe_endpoint(cluster_name, wait_master_fe_query_addr_file=False):
- cluster_path = get_cluster_path(cluster_name)
- if os.path.exists(cluster_path):
+ if os.path.exists(Cluster._get_meta_file(cluster_name)):
master_fe_query_addr_file = get_master_fe_addr_path(cluster_name)
max_retries = 10 if wait_master_fe_query_addr_file else 0
i = 0
@@ -464,6 +463,7 @@ class Node(object):
"volumes": volumes,
}
+ extra_hosts = []
if self.cluster.is_host_network():
content["network_mode"] = "host"
else:
@@ -473,11 +473,16 @@ class Node(object):
"ipv4_address": self.get_ip(),
}
}
- content["extra_hosts"] = [
+ extra_hosts.extend([
"{}:{}".format(node.get_name(), node.get_ip())
for node in self.cluster.get_all_nodes()
- ]
+ ])
content["ports"] = self.docker_ports()
+ user_hosts = getattr(self.cluster, "extra_hosts", [])
+ if user_hosts:
+ extra_hosts.extend(user_hosts)
+ if extra_hosts:
+ content["extra_hosts"] = extra_hosts
if self.entrypoint():
content["entrypoint"] = self.entrypoint()
@@ -780,8 +785,8 @@ class Cluster(object):
def __init__(self, name, subnet, image, is_cloud, is_root_user, fe_config,
be_config, ms_config, recycle_config, remote_master_fe,
local_network_ip, fe_follower, be_disks, be_cluster, reg_be,
- coverage_dir, cloud_store_config, sql_mode_node_mgr,
- be_metaservice_endpoint, be_cluster_id):
+ extra_hosts, coverage_dir, cloud_store_config,
+ sql_mode_node_mgr, be_metaservice_endpoint, be_cluster_id):
self.name = name
self.subnet = subnet
self.image = image
@@ -797,6 +802,7 @@ class Cluster(object):
self.be_disks = be_disks
self.be_cluster = be_cluster
self.reg_be = reg_be
+ self.extra_hosts = extra_hosts
self.coverage_dir = coverage_dir
self.cloud_store_config = cloud_store_config
self.groups = {
@@ -813,9 +819,9 @@ class Cluster(object):
@staticmethod
def new(name, image, is_cloud, is_root_user, fe_config, be_config,
ms_config, recycle_config, remote_master_fe, local_network_ip,
- fe_follower, be_disks, be_cluster, reg_be, coverage_dir,
- cloud_store_config, sql_mode_node_mgr, be_metaservice_endpoint,
- be_cluster_id):
+ fe_follower, be_disks, be_cluster, reg_be, extra_hosts,
+ coverage_dir, cloud_store_config, sql_mode_node_mgr,
+ be_metaservice_endpoint, be_cluster_id):
if not os.path.exists(LOCAL_DORIS_PATH):
os.makedirs(LOCAL_DORIS_PATH, exist_ok=True)
os.chmod(LOCAL_DORIS_PATH, 0o777)
@@ -827,9 +833,10 @@ class Cluster(object):
cluster = Cluster(name, subnet, image, is_cloud, is_root_user,
fe_config, be_config, ms_config, recycle_config,
remote_master_fe, local_network_ip, fe_follower,
- be_disks, be_cluster, reg_be, coverage_dir,
- cloud_store_config, sql_mode_node_mgr,
- be_metaservice_endpoint, be_cluster_id)
+ be_disks, be_cluster, reg_be, extra_hosts,
+ coverage_dir, cloud_store_config,
+ sql_mode_node_mgr, be_metaservice_endpoint,
+ be_cluster_id)
os.makedirs(cluster.get_path(), exist_ok=True)
os.makedirs(get_status_path(name), exist_ok=True)
cluster._save_meta()
@@ -860,7 +867,7 @@ class Cluster(object):
return os.path.join(get_cluster_path(name), "meta")
def is_host_network(self):
- return getattr(self, "remote_master_fe", "")
+ return self.remote_master_fe
def get_remote_fe_node(self):
if not self.is_host_network():
diff --git a/docker/runtime/doris-compose/command.py
b/docker/runtime/doris-compose/command.py
index f3e3754464e..1194b3d1f15 100644
--- a/docker/runtime/doris-compose/command.py
+++ b/docker/runtime/doris-compose/command.py
@@ -430,6 +430,14 @@ class UpCommand(Command):
help="Recreate containers even if their
configuration " \
"and image haven't changed. ")
+ parser.add_argument(
+ "--extra-hosts",
+ nargs="*",
+ type=str,
+ help=
+ "Add custom host-to-IP mappings (host:ip). For example:
--extra-hosts myhost1:192.168.10.1 myhost2:192.168.10.2 . Only use when
creating new cluster."
+ )
+
parser.add_argument("--coverage-dir",
default="",
help="Set code coverage output directory")
@@ -599,8 +607,8 @@ class UpCommand(Command):
args.NAME, args.IMAGE, args.cloud, args.root, args.fe_config,
args.be_config, args.ms_config, args.recycle_config,
args.remote_master_fe, args.local_network_ip, args.fe_follower,
- args.be_disks, args.be_cluster, args.reg_be, args.coverage_dir,
- cloud_store_config, args.sql_mode_node_mgr,
+ args.be_disks, args.be_cluster, args.reg_be, args.extra_hosts,
+ args.coverage_dir, cloud_store_config, args.sql_mode_node_mgr,
args.be_metaservice_endpoint, args.be_cluster_id)
LOG.info("Create new cluster {} succ, cluster path is {}".format(
args.NAME, cluster.get_path()))
diff --git
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy
index bf43a57fbf7..ec801c47c01 100644
---
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy
+++
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy
@@ -50,6 +50,7 @@ class ClusterOptions {
// for example, ' xx = yy ' is bad, should use 'xx=yy'
List<String> feConfigs = [
'heartbeat_interval_second=5',
+ 'workload_group_check_interval_ms=1000',
]
// don't add whitespace in beConfigs items,
@@ -64,6 +65,10 @@ class ClusterOptions {
List<String> recycleConfigs = []
+ // host mapping(host:IP), for example: myhost:192.168.10.10
+ // just as `docker run --add-host myhost:192.168.10.10` do.
+ List<String> extraHosts = []
+
boolean connectToFollower = false
// 1. cloudMode = true, only create cloud cluster.
@@ -306,19 +311,19 @@ class SuiteCluster {
cmd += ['--add-ms-num', String.valueOf(options.msNum)]
}
// TODO: need escape white space in config
- if (options.feConfigs != null && options.feConfigs.size() > 0) {
+ if (!options.feConfigs.isEmpty()) {
cmd += ['--fe-config']
cmd += options.feConfigs
}
- if (options.beConfigs != null && options.beConfigs.size() > 0) {
+ if (!options.beConfigs.isEmpty()) {
cmd += ['--be-config']
cmd += options.beConfigs
}
- if (options.msConfigs != null && options.msConfigs.size() > 0) {
+ if (!options.msConfigs.isEmpty()) {
cmd += ['--ms-config']
cmd += options.msConfigs
}
- if (options.recycleConfigs != null && options.recycleConfigs.size() >
0) {
+ if (!options.recycleConfigs.isEmpty()) {
cmd += ['--recycle-config']
cmd += options.recycleConfigs
}
@@ -326,6 +331,10 @@ class SuiteCluster {
cmd += ['--be-disks']
cmd += options.beDisks
}
+ if (!options.extraHosts.isEmpty()) {
+ cmd += ['--extra-hosts']
+ cmd += options.extraHosts
+ }
if (config.dockerCoverageOutputDir != null &&
config.dockerCoverageOutputDir != '') {
cmd += ['--coverage-dir', config.dockerCoverageOutputDir]
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
