This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 68c28a0ee5c [feature](doris compose) diff user use diff start subnet
(#41996)
68c28a0ee5c is described below
commit 68c28a0ee5c15421807fcd350d627e8d0eb8751b
Author: yujun <[email protected]>
AuthorDate: Sun Oct 20 10:57:57 2024 +0800
[feature](doris compose) diff user use diff start subnet (#41996)
If users use different LOCAL_DORIS_PATH, their clusters' network maybe
conflict. So let different user use different searched start subnet.
---
docker/runtime/doris-compose/cluster.py | 45 +++++++++++++++++++---
docker/runtime/doris-compose/command.py | 2 +-
docker/runtime/doris-compose/resource/init_be.sh | 2 +-
.../runtime/doris-compose/resource/init_cloud.sh | 5 +--
docker/runtime/doris-compose/resource/init_fe.sh | 15 +++++---
5 files changed, 53 insertions(+), 16 deletions(-)
diff --git a/docker/runtime/doris-compose/cluster.py
b/docker/runtime/doris-compose/cluster.py
index 985ef27113b..6285e4c615c 100644
--- a/docker/runtime/doris-compose/cluster.py
+++ b/docker/runtime/doris-compose/cluster.py
@@ -16,6 +16,8 @@
# under the License.
import filelock
+import getpass
+import hashlib
import jsonpickle
import os
import os.path
@@ -23,7 +25,10 @@ import utils
DOCKER_DORIS_PATH = "/opt/apache-doris"
LOCAL_DORIS_PATH = os.getenv("LOCAL_DORIS_PATH", "/tmp/doris")
-DORIS_SUBNET_START = int(os.getenv("DORIS_SUBNET_START", 128))
+
+# an integer between 128 and 191, generally no need to set
+DORIS_SUBNET_START = os.getenv("DORIS_SUBNET_START")
+
LOCAL_RESOURCE_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)),
"resource")
DOCKER_RESOURCE_PATH = os.path.join(DOCKER_DORIS_PATH, "resource")
@@ -93,11 +98,39 @@ def gen_subnet_prefix16():
except:
pass
- for i in range(DORIS_SUBNET_START, 192):
- for j in range(256):
- subnet = "{}.{}".format(i, j)
- if not used_subnet.get(subnet, False):
- return subnet
+ subnet_begin = 128
+ subnet_end = 192
+
+ subnet_part_1 = None
+ subnet_part_2 = None
+ if DORIS_SUBNET_START:
+ subnet_part_1 = int(DORIS_SUBNET_START)
+ subnet_part_2 = 0
+ else:
+ m = hashlib.md5()
+ m.update(getpass.getuser().encode("utf-8"))
+ hash_val = int(m.hexdigest(), 16)
+ # want subnet part ii to be a small num, just less than 100, so don't
use 256 here.
+ small_width = 100
+ slot_num = (subnet_end - subnet_begin) * small_width
+ idx = hash_val % slot_num
+ if idx < 0:
+ idx += slot_num
+ subnet_part_1 = subnet_begin + int(idx / small_width)
+ subnet_part_2 = idx % small_width
+
+ intervals = [
+ [(subnet_part_1, subnet_part_1 + 1), (subnet_part_2, 256)],
+ [(subnet_part_1 + 1, subnet_end), (0, 256)],
+ [(subnet_begin, subnet_part_1), (0, 256)],
+ [(subnet_part_1, subnet_part_1 + 1), (0, subnet_part_2)],
+ ]
+ for interval in intervals:
+ for i in range(interval[0][0], interval[0][1]):
+ for j in range(interval[1][0], interval[1][1]):
+ subnet = "{}.{}".format(i, j)
+ if not used_subnet.get(subnet, False):
+ return subnet
raise Exception("Failed to gen subnet")
diff --git a/docker/runtime/doris-compose/command.py
b/docker/runtime/doris-compose/command.py
index 48863003223..cffc76df7c7 100644
--- a/docker/runtime/doris-compose/command.py
+++ b/docker/runtime/doris-compose/command.py
@@ -294,7 +294,7 @@ class UpCommand(Command):
group2.add_argument("--force-recreate",
default=False,
action=self._get_parser_bool_action(True),
- help="Recreate containers even if their
configuration" \
+ help="Recreate containers even if their
configuration " \
"and image haven't changed. ")
parser.add_argument("--coverage-dir",
diff --git a/docker/runtime/doris-compose/resource/init_be.sh
b/docker/runtime/doris-compose/resource/init_be.sh
index e0ed5b92cb8..08cc914f6af 100755
--- a/docker/runtime/doris-compose/resource/init_be.sh
+++ b/docker/runtime/doris-compose/resource/init_be.sh
@@ -173,7 +173,7 @@ main() {
add_be_to_cluster
health_log "run start_be.sh"
- bash $DORIS_HOME/bin/start_be.sh --daemon
+ bash $DORIS_HOME/bin/start_be.sh --daemon | tee -a $DORIS_HOME/log/be.out
wait_process
}
diff --git a/docker/runtime/doris-compose/resource/init_cloud.sh
b/docker/runtime/doris-compose/resource/init_cloud.sh
index 5740335ace3..18dfc4430e2 100644
--- a/docker/runtime/doris-compose/resource/init_cloud.sh
+++ b/docker/runtime/doris-compose/resource/init_cloud.sh
@@ -116,9 +116,8 @@ main() {
check_init_cloud &
- health_log "input args: $ARGS"
-
- bash bin/start.sh $ARGS --daemon
+ health_log "run starts.sh with args: $ARGS"
+ bash bin/start.sh $ARGS --daemon | tee -a $DORIS_HOME/log/doris_cloud.out
wait_process
}
diff --git a/docker/runtime/doris-compose/resource/init_fe.sh
b/docker/runtime/doris-compose/resource/init_fe.sh
index 39d3ed3fa93..b69ac3a209e 100755
--- a/docker/runtime/doris-compose/resource/init_fe.sh
+++ b/docker/runtime/doris-compose/resource/init_fe.sh
@@ -81,10 +81,15 @@ fe_daemon() {
done
}
+run_fe() {
+ health_log "run start_fe.sh"
+ bash $DORIS_HOME/bin/start_fe.sh --daemon $@ | tee -a
$DORIS_HOME/log/fe.out
+}
+
start_cloud_fe() {
if [ -f "$REGISTER_FILE" ]; then
fe_daemon &
- bash $DORIS_HOME/bin/start_fe.sh --daemon
+ run_fe
return
fi
@@ -95,7 +100,7 @@ start_cloud_fe() {
touch $REGISTER_FILE
fe_daemon &
- bash $DORIS_HOME/bin/start_fe.sh --daemon
+ run_fe
if [ "$MY_ID" == "1" ]; then
echo $MY_IP >$MASTER_FE_IP_FILE
@@ -162,7 +167,7 @@ start_cloud_fe() {
touch $REGISTER_FILE
fe_daemon &
- bash $DORIS_HOME/bin/start_fe.sh --daemon
+ run_fe
if [ "$MY_ID" == "1" ]; then
echo $MY_IP >$MASTER_FE_IP_FILE
@@ -199,11 +204,11 @@ start_local_fe() {
if [ -f $REGISTER_FILE ]; then
fe_daemon &
- bash $DORIS_HOME/bin/start_fe.sh --daemon
+ run_fe
else
add_local_fe
fe_daemon &
- bash $DORIS_HOME/bin/start_fe.sh --helper
$MASTER_FE_IP:$FE_EDITLOG_PORT --daemon
+ run_fe --helper $MASTER_FE_IP:$FE_EDITLOG_PORT
fi
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
