This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/dev/pr-12710-14a27ac26a0a71b56722f86cdf8ec2779094ceaf in repository https://gitbox.apache.org/repos/asf/seatunnel.git
commit 42cbb056d1962cd469bc42a98f79e3b2a559546c Author: Ricky Makhija <[email protected]> AuthorDate: Fri Oct 23 13:33:15 2026 +0000 [Fix][Zeta] Exec the JVM in seatunnel-cluster.sh so SIGTERM reaches it (#12710) Co-authored-by: David Zollo <[email protected]> --- .../templates/deployment-seatunnel-master.yaml | 5 +++- .../templates/deployment-seatunnel-worker.yaml | 5 +++- deploy/kubernetes/seatunnel/values.yaml | 6 ++++ docs/en/engines/zeta/tuning-guide.md | 2 ++ docs/en/getting-started/docker/docker.md | 35 +++++----------------- .../kubernetes/hybrid-cluster-mode.md | 18 +++++++++-- docs/en/getting-started/kubernetes/operations.md | 3 +- .../kubernetes/separated-cluster-mode.md | 18 +++++++++-- docs/zh/engines/zeta/tuning-guide.md | 2 ++ docs/zh/getting-started/docker/docker.md | 35 +++++----------------- .../kubernetes/hybrid-cluster-mode.md | 18 +++++++++-- docs/zh/getting-started/kubernetes/operations.md | 3 +- .../kubernetes/separated-cluster-mode.md | 18 +++++++++-- .../src/main/bin/seatunnel-cluster.sh | 3 +- .../seatunnel/SeaTunnelClusterScriptTest.java | 19 ++++++++++++ 15 files changed, 121 insertions(+), 69 deletions(-) diff --git a/deploy/kubernetes/seatunnel/templates/deployment-seatunnel-master.yaml b/deploy/kubernetes/seatunnel/templates/deployment-seatunnel-master.yaml index f6d904410a..f898052f69 100644 --- a/deploy/kubernetes/seatunnel/templates/deployment-seatunnel-master.yaml +++ b/deploy/kubernetes/seatunnel/templates/deployment-seatunnel-master.yaml @@ -40,6 +40,9 @@ spec: {{- include "seatunnel.master.labels" . | nindent 8 }} spec: serviceAccountName: {{ template "seatunnel.fullname" . }} + {{- if .Values.master.terminationGracePeriodSeconds }} + terminationGracePeriodSeconds: {{ .Values.master.terminationGracePeriodSeconds }} + {{- end }} {{- if .Values.master.affinity }} affinity: {{- toYaml .Values.master.affinity | nindent 8 }} @@ -68,7 +71,7 @@ spec: {{- if .Values.master.command }} command: {{ .Values.master.command }} {{- else }} - command: ["/bin/sh","-c","/opt/seatunnel/bin/seatunnel-cluster.sh -r master"] + command: ["/opt/seatunnel/bin/seatunnel-cluster.sh","-r","master"] {{- end }} {{- if .Values.master.resources }} resources: diff --git a/deploy/kubernetes/seatunnel/templates/deployment-seatunnel-worker.yaml b/deploy/kubernetes/seatunnel/templates/deployment-seatunnel-worker.yaml index 612db0c702..81899fa9b5 100644 --- a/deploy/kubernetes/seatunnel/templates/deployment-seatunnel-worker.yaml +++ b/deploy/kubernetes/seatunnel/templates/deployment-seatunnel-worker.yaml @@ -40,6 +40,9 @@ spec: {{- include "seatunnel.worker.labels" . | nindent 8 }} spec: serviceAccountName: {{ template "seatunnel.fullname" . }} + {{- if .Values.worker.terminationGracePeriodSeconds }} + terminationGracePeriodSeconds: {{ .Values.worker.terminationGracePeriodSeconds }} + {{- end }} {{- if .Values.worker.affinity }} affinity: {{- toYaml .Values.worker.affinity | nindent 8 }} @@ -66,7 +69,7 @@ spec: {{- if .Values.worker.command }} command: {{ .Values.worker.command }} {{- else }} - command: ["/bin/sh","-c","/opt/seatunnel/bin/seatunnel-cluster.sh -r worker"] + command: ["/opt/seatunnel/bin/seatunnel-cluster.sh","-r","worker"] {{- end }} {{- if .Values.worker.resources }} resources: diff --git a/deploy/kubernetes/seatunnel/values.yaml b/deploy/kubernetes/seatunnel/values.yaml index d8c7cc387b..e98a9a454f 100644 --- a/deploy/kubernetes/seatunnel/values.yaml +++ b/deploy/kubernetes/seatunnel/values.yaml @@ -43,6 +43,9 @@ env: master: ## The command to start master. command: [] + ## Seconds Kubernetes waits after SIGTERM before SIGKILL. With -Dhazelcast.shutdownhook.policy=GRACEFUL, + ## keep hazelcast.graceful.shutdown.max.wait below this value. 0 is treated as unset (Kubernetes default). + terminationGracePeriodSeconds: 30 ## The deployment strategy to use to replace existing pods with new ones. strategy: type: RollingUpdate @@ -106,6 +109,9 @@ master: worker: ## The command to start worker. command: [] + ## Seconds Kubernetes waits after SIGTERM before SIGKILL. With -Dhazelcast.shutdownhook.policy=GRACEFUL, + ## keep hazelcast.graceful.shutdown.max.wait below this value. 0 is treated as unset (Kubernetes default). + terminationGracePeriodSeconds: 30 ## The deployment strategy to use to replace existing pods with new ones. strategy: type: RollingUpdate diff --git a/docs/en/engines/zeta/tuning-guide.md b/docs/en/engines/zeta/tuning-guide.md index fa07df7057..c49b2b5ee9 100644 --- a/docs/en/engines/zeta/tuning-guide.md +++ b/docs/en/engines/zeta/tuning-guide.md @@ -496,6 +496,8 @@ Ensure pods have enough time to flush state and leave the cluster: terminationGracePeriodSeconds: 60 ``` +The JVM receives SIGTERM only when the container runs `seatunnel-cluster.sh` directly, not through `/bin/sh -c`. See [Health Check and Shutdown](../../getting-started/kubernetes/hybrid-cluster-mode.md#health-check-and-shutdown). + And in `hazelcast.yaml`: ```yaml hazelcast: diff --git a/docs/en/getting-started/docker/docker.md b/docs/en/getting-started/docker/docker.md index 5a47f0c1f8..fcd4959b1a 100644 --- a/docs/en/getting-started/docker/docker.md +++ b/docs/en/getting-started/docker/docker.md @@ -233,10 +233,7 @@ services: restart: unless-stopped environment: - ST_DOCKER_MEMBER_LIST=seatunnel_master:5801,seatunnel_worker_1:5801,seatunnel_worker_2:5801 - entrypoint: > - /bin/sh -c " - /opt/seatunnel/bin/seatunnel-cluster.sh -r master - " + entrypoint: ["/opt/seatunnel/bin/seatunnel-cluster.sh", "-r", "master"] ports: - "5801:5801" networks: @@ -248,10 +245,7 @@ services: restart: unless-stopped environment: - ST_DOCKER_MEMBER_LIST=seatunnel_master:5801,seatunnel_worker_1:5801,seatunnel_worker_2:5801 - entrypoint: > - /bin/sh -c " - /opt/seatunnel/bin/seatunnel-cluster.sh -r worker - " + entrypoint: ["/opt/seatunnel/bin/seatunnel-cluster.sh", "-r", "worker"] depends_on: - master networks: @@ -263,10 +257,7 @@ services: restart: unless-stopped environment: - ST_DOCKER_MEMBER_LIST=seatunnel_master:5801,seatunnel_worker_1:5801,seatunnel_worker_2:5801 - entrypoint: > - /bin/sh -c " - /opt/seatunnel/bin/seatunnel-cluster.sh -r worker - " + entrypoint: ["/opt/seatunnel/bin/seatunnel-cluster.sh", "-r", "worker"] depends_on: - master networks: @@ -300,10 +291,7 @@ services: restart: unless-stopped environment: - ST_DOCKER_MEMBER_LIST=seatunnel_master:5801,seatunnel_worker_1:5801,seatunnel_worker_2:5801 - entrypoint: > - /bin/sh -c " - /opt/seatunnel/bin/seatunnel-cluster.sh -r master - " + entrypoint: ["/opt/seatunnel/bin/seatunnel-cluster.sh", "-r", "master"] ports: - "5801:5801" networks: @@ -315,10 +303,7 @@ services: restart: unless-stopped environment: - ST_DOCKER_MEMBER_LIST=seatunnel_master:5801,seatunnel_worker_1:5801,seatunnel_worker_2:5801 - entrypoint: > - /bin/sh -c " - /opt/seatunnel/bin/seatunnel-cluster.sh -r worker - " + entrypoint: ["/opt/seatunnel/bin/seatunnel-cluster.sh", "-r", "worker"] depends_on: - master networks: @@ -330,10 +315,7 @@ services: restart: unless-stopped environment: - ST_DOCKER_MEMBER_LIST=seatunnel_master:5801,seatunnel_worker_1:5801,seatunnel_worker_2:5801 - entrypoint: > - /bin/sh -c " - /opt/seatunnel/bin/seatunnel-cluster.sh -r worker - " + entrypoint: ["/opt/seatunnel/bin/seatunnel-cluster.sh", "-r", "worker"] depends_on: - master networks: @@ -348,10 +330,7 @@ services: restart: unless-stopped environment: - ST_DOCKER_MEMBER_LIST=seatunnel_master:5801,seatunnel_worker_1:5801,seatunnel_worker_2:5801,seatunnel_worker_3:5801 - entrypoint: > - /bin/sh -c " - /opt/seatunnel/bin/seatunnel-cluster.sh -r worker - " + entrypoint: ["/opt/seatunnel/bin/seatunnel-cluster.sh", "-r", "worker"] depends_on: - master networks: diff --git a/docs/en/getting-started/kubernetes/hybrid-cluster-mode.md b/docs/en/getting-started/kubernetes/hybrid-cluster-mode.md index e065e37a1f..0b74f32516 100644 --- a/docs/en/getting-started/kubernetes/hybrid-cluster-mode.md +++ b/docs/en/getting-started/kubernetes/hybrid-cluster-mode.md @@ -281,9 +281,9 @@ spec: name: seatunnel-engine-config ``` -## Health Check and Graceful Shutdown +## Health Check and Shutdown -For cluster mode, add `startupProbe`, health checks, and `preStop` to each SeaTunnel container to avoid killing slow starts and to reduce the impact of rolling updates or node eviction. +For cluster mode, add `startupProbe` and health checks to each SeaTunnel container to avoid killing slow starts and to reduce the impact of rolling updates or node eviction. ```yaml startupProbe: @@ -305,6 +305,19 @@ livenessProbe: periodSeconds: 30 timeoutSeconds: 5 failureThreshold: 3 +``` + +On pod deletion Kubernetes sends SIGTERM to PID 1 and sends SIGKILL after `terminationGracePeriodSeconds`. The StatefulSet above already sets the two things a clean exit needs: + +- `command` runs `seatunnel-cluster.sh` directly, not through `/bin/sh -c`. The script `exec`s the JVM, so the JVM is PID 1 and receives the SIGTERM. A `/bin/sh -c` wrapper as PID 1 ignores SIGTERM, and the pod is killed only after the grace period. +- `terminationGracePeriodSeconds` gives the JVM time to run its shutdown hook before SIGKILL. + +On SIGTERM the Hazelcast shutdown hook runs with `hazelcast.shutdownhook.policy`, the node stops its running tasks and leaves the cluster within seconds, and those tasks recover from their last checkpoint. This is a fast, clean exit from the cluster, not a graceful stop of running jobs. `GRACEFUL` only lets Hazelcast migrate partition data before the member leaves; if you use it, keep `hazelcast.graceful.shutdown.max.wait` below `terminationGracePeriodSeconds`. + +:::tip Optional preStop +A `preStop` hook is only needed when the container cannot run the script directly, for example when `command` must stay `/bin/sh -c`. `stop-seatunnel-cluster.sh` sends SIGTERM to the `SeaTunnelServer` process itself, and the hook then waits for the process to exit: + +```yaml lifecycle: preStop: exec: @@ -317,6 +330,7 @@ lifecycle: sleep 1 done ``` +::: Apply the manifests: diff --git a/docs/en/getting-started/kubernetes/operations.md b/docs/en/getting-started/kubernetes/operations.md index aefdd24a80..914863457c 100644 --- a/docs/en/getting-started/kubernetes/operations.md +++ b/docs/en/getting-started/kubernetes/operations.md @@ -72,8 +72,9 @@ Do not scale down multiple Workers at once. Decrease replicas one by one and obs When image or configuration changes, StatefulSet updates Pods in order. Recommended practices: -- Configure `preStop` to call `stop-seatunnel-cluster.sh`. +- Run `seatunnel-cluster.sh` directly in the container `command`, not through `/bin/sh -c`, so the JVM receives SIGTERM. - Set a long enough `terminationGracePeriodSeconds`. +- `preStop` with `stop-seatunnel-cluster.sh` is optional, only needed when the command must stay `/bin/sh -c`. See [Health Check and Shutdown](hybrid-cluster-mode.md#health-check-and-shutdown). - Check Master replicas and Worker slot capacity before updates. - Avoid updating Master and Worker at the same time during peak hours. - If configuration files are mounted through `subPath`, ConfigMap updates are not propagated into the running container automatically. Run a rolling restart or use a tool such as Reloader to trigger restarts. diff --git a/docs/en/getting-started/kubernetes/separated-cluster-mode.md b/docs/en/getting-started/kubernetes/separated-cluster-mode.md index e522144e2a..a0dc221a6f 100644 --- a/docs/en/getting-started/kubernetes/separated-cluster-mode.md +++ b/docs/en/getting-started/kubernetes/separated-cluster-mode.md @@ -446,9 +446,9 @@ spec: Custom plugins are not included in the base StatefulSet. If extra plugins are required, see [Plugin Loading](configuration.md#plugin-loading) and manage them separately with a custom image, initContainer overlay, PersistentVolume, or object storage. -## Health Check and Graceful Shutdown +## Health Check and Shutdown -Add the following container fragment to both Master and Worker. `startupProbe` avoids killing slow starts. `preStop` calls the SeaTunnel stop script first, then waits for the `SeaTunnelServer` process to exit. Together with `terminationGracePeriodSeconds`, this reduces the impact of rolling updates and node eviction on running jobs. +Add the following probes to both Master and Worker. `startupProbe` avoids killing slow starts. ```yaml startupProbe: @@ -470,6 +470,19 @@ livenessProbe: periodSeconds: 30 timeoutSeconds: 5 failureThreshold: 3 +``` + +On pod deletion Kubernetes sends SIGTERM to PID 1 and sends SIGKILL after `terminationGracePeriodSeconds`. The Master and Worker StatefulSets above already set the two things a clean exit needs: + +- `command` runs `seatunnel-cluster.sh` directly, not through `/bin/sh -c`. The script `exec`s the JVM, so the JVM is PID 1 and receives the SIGTERM. A `/bin/sh -c` wrapper as PID 1 ignores SIGTERM, and the pod is killed only after the grace period. +- `terminationGracePeriodSeconds` gives the JVM time to run its shutdown hook before SIGKILL. + +On SIGTERM the Hazelcast shutdown hook runs with `hazelcast.shutdownhook.policy`, the node stops its running tasks and leaves the cluster within seconds, and those tasks recover from their last checkpoint. This is a fast, clean exit from the cluster, not a graceful stop of running jobs. `GRACEFUL` only lets Hazelcast migrate partition data before the member leaves; if you use it, keep `hazelcast.graceful.shutdown.max.wait` below `terminationGracePeriodSeconds`. + +:::tip Optional preStop +A `preStop` hook is only needed when the container cannot run the script directly, for example when `command` must stay `/bin/sh -c`. `stop-seatunnel-cluster.sh` sends SIGTERM to the `SeaTunnelServer` process itself, and the hook then waits for the process to exit: + +```yaml lifecycle: preStop: exec: @@ -482,6 +495,7 @@ lifecycle: sleep 1 done ``` +::: ## Apply Manifests diff --git a/docs/zh/engines/zeta/tuning-guide.md b/docs/zh/engines/zeta/tuning-guide.md index 40505bcbbe..5e2eb104a7 100644 --- a/docs/zh/engines/zeta/tuning-guide.md +++ b/docs/zh/engines/zeta/tuning-guide.md @@ -495,6 +495,8 @@ readinessProbe: terminationGracePeriodSeconds: 60 ``` +只有当容器直接运行 `seatunnel-cluster.sh`（不通过 `/bin/sh -c` 包装）时，JVM 才能收到 SIGTERM。参见 [健康检查和停止](../../getting-started/kubernetes/hybrid-cluster-mode.md#健康检查和停止)。 + 并在 `hazelcast.yaml` 中： ```yaml hazelcast: diff --git a/docs/zh/getting-started/docker/docker.md b/docs/zh/getting-started/docker/docker.md index 7b6d2694cd..d5861a6107 100644 --- a/docs/zh/getting-started/docker/docker.md +++ b/docs/zh/getting-started/docker/docker.md @@ -231,10 +231,7 @@ services: restart: unless-stopped environment: - ST_DOCKER_MEMBER_LIST=seatunnel_master:5801,seatunnel_worker_1:5801,seatunnel_worker_2:5801 - entrypoint: > - /bin/sh -c " - /opt/seatunnel/bin/seatunnel-cluster.sh -r master - " + entrypoint: ["/opt/seatunnel/bin/seatunnel-cluster.sh", "-r", "master"] ports: - "5801:5801" networks: @@ -246,10 +243,7 @@ services: restart: unless-stopped environment: - ST_DOCKER_MEMBER_LIST=seatunnel_master:5801,seatunnel_worker_1:5801,seatunnel_worker_2:5801 - entrypoint: > - /bin/sh -c " - /opt/seatunnel/bin/seatunnel-cluster.sh -r worker - " + entrypoint: ["/opt/seatunnel/bin/seatunnel-cluster.sh", "-r", "worker"] depends_on: - master networks: @@ -261,10 +255,7 @@ services: restart: unless-stopped environment: - ST_DOCKER_MEMBER_LIST=seatunnel_master:5801,seatunnel_worker_1:5801,seatunnel_worker_2:5801 - entrypoint: > - /bin/sh -c " - /opt/seatunnel/bin/seatunnel-cluster.sh -r worker - " + entrypoint: ["/opt/seatunnel/bin/seatunnel-cluster.sh", "-r", "worker"] depends_on: - master networks: @@ -293,10 +284,7 @@ services: restart: unless-stopped environment: - ST_DOCKER_MEMBER_LIST=seatunnel_master:5801,seatunnel_worker_1:5801,seatunnel_worker_2:5801 - entrypoint: > - /bin/sh -c " - /opt/seatunnel/bin/seatunnel-cluster.sh -r master - " + entrypoint: ["/opt/seatunnel/bin/seatunnel-cluster.sh", "-r", "master"] ports: - "5801:5801" networks: @@ -308,10 +296,7 @@ services: restart: unless-stopped environment: - ST_DOCKER_MEMBER_LIST=seatunnel_master:5801,seatunnel_worker_1:5801,seatunnel_worker_2:5801 - entrypoint: > - /bin/sh -c " - /opt/seatunnel/bin/seatunnel-cluster.sh -r worker - " + entrypoint: ["/opt/seatunnel/bin/seatunnel-cluster.sh", "-r", "worker"] depends_on: - master networks: @@ -323,10 +308,7 @@ services: restart: unless-stopped environment: - ST_DOCKER_MEMBER_LIST=seatunnel_master:5801,seatunnel_worker_1:5801,seatunnel_worker_2:5801 - entrypoint: > - /bin/sh -c " - /opt/seatunnel/bin/seatunnel-cluster.sh -r worker - " + entrypoint: ["/opt/seatunnel/bin/seatunnel-cluster.sh", "-r", "worker"] depends_on: - master networks: @@ -341,10 +323,7 @@ services: restart: unless-stopped environment: - ST_DOCKER_MEMBER_LIST=seatunnel_master:5801,seatunnel_worker_1:5801,seatunnel_worker_2:5801,seatunnel_worker_3:5801 - entrypoint: > - /bin/sh -c " - /opt/seatunnel/bin/seatunnel-cluster.sh -r worker - " + entrypoint: ["/opt/seatunnel/bin/seatunnel-cluster.sh", "-r", "worker"] depends_on: - master networks: diff --git a/docs/zh/getting-started/kubernetes/hybrid-cluster-mode.md b/docs/zh/getting-started/kubernetes/hybrid-cluster-mode.md index 16cb49d66b..cace5b4ec3 100644 --- a/docs/zh/getting-started/kubernetes/hybrid-cluster-mode.md +++ b/docs/zh/getting-started/kubernetes/hybrid-cluster-mode.md @@ -281,9 +281,9 @@ spec: name: seatunnel-engine-config ``` -## 健康检查和优雅停止 +## 健康检查和停止 -集群模式建议为每个 SeaTunnel 容器添加 `startupProbe`、健康检查和 `preStop`，避免启动期误杀，并减少滚动更新或节点驱逐对集群的影响。 +集群模式建议为每个 SeaTunnel 容器添加 `startupProbe` 和健康检查，避免启动期误杀，并减少滚动更新或节点驱逐对集群的影响。 ```yaml startupProbe: @@ -305,6 +305,19 @@ livenessProbe: periodSeconds: 30 timeoutSeconds: 5 failureThreshold: 3 +``` + +删除 Pod 时，Kubernetes 向 PID 1 发送 SIGTERM，并在 `terminationGracePeriodSeconds` 后发送 SIGKILL。上面的 StatefulSet 已经包含干净退出所需的两项配置： + +- `command` 直接运行 `seatunnel-cluster.sh`，不通过 `/bin/sh -c` 包装。脚本会 `exec` 启动 JVM，使 JVM 成为 PID 1 并收到 SIGTERM。若 PID 1 是 `/bin/sh -c`，SIGTERM 会被忽略，Pod 要等到宽限期结束后才被强制终止。 +- `terminationGracePeriodSeconds` 让 JVM 在 SIGKILL 之前有时间执行关闭钩子。 + +收到 SIGTERM 后，Hazelcast 会按 `hazelcast.shutdownhook.policy` 执行关闭钩子，节点停止其上运行的任务并在数秒内离开集群，这些任务从最近的 checkpoint 恢复。这是快速、干净地退出集群，而不是优雅地停止运行中的作业。`GRACEFUL` 只会让 Hazelcast 在成员离开前迁移分区数据；若使用 `GRACEFUL`，请确保 `hazelcast.graceful.shutdown.max.wait` 小于 `terminationGracePeriodSeconds`。 + +:::tip 可选的 preStop +只有当容器无法直接运行脚本时（例如 `command` 必须保留 `/bin/sh -c`）才需要 `preStop`。`stop-seatunnel-cluster.sh` 会直接向 `SeaTunnelServer` 进程发送 SIGTERM，随后钩子等待该进程退出： + +```yaml lifecycle: preStop: exec: @@ -317,6 +330,7 @@ lifecycle: sleep 1 done ``` +::: 应用配置： diff --git a/docs/zh/getting-started/kubernetes/operations.md b/docs/zh/getting-started/kubernetes/operations.md index f65adde83c..83f630b242 100644 --- a/docs/zh/getting-started/kubernetes/operations.md +++ b/docs/zh/getting-started/kubernetes/operations.md @@ -72,8 +72,9 @@ kubectl get pods -l app=seatunnel,component=worker 更新镜像或配置时，StatefulSet 会按顺序滚动更新 Pod。建议： -- 配置 `preStop` 调用 `stop-seatunnel-cluster.sh`。 +- 容器 `command` 直接运行 `seatunnel-cluster.sh`，不要通过 `/bin/sh -c` 包装，使 JVM 能收到 SIGTERM。 - 设置足够长的 `terminationGracePeriodSeconds`。 +- 通过 `preStop` 调用 `stop-seatunnel-cluster.sh` 是可选的，仅在 `command` 必须保留 `/bin/sh -c` 时需要。参见 [健康检查和停止](hybrid-cluster-mode.md#健康检查和停止)。 - 更新前确认 Master 副本数和 Worker slot 余量。 - 避免在高峰期同时更新 Master 和 Worker。 - 如果配置文件通过 `subPath` 挂载，ConfigMap 更新不会自动同步到容器内，需要执行滚动重启或通过 Reloader 等工具触发重启。 diff --git a/docs/zh/getting-started/kubernetes/separated-cluster-mode.md b/docs/zh/getting-started/kubernetes/separated-cluster-mode.md index fbb6f87ff7..ea63161119 100644 --- a/docs/zh/getting-started/kubernetes/separated-cluster-mode.md +++ b/docs/zh/getting-started/kubernetes/separated-cluster-mode.md @@ -446,9 +446,9 @@ spec: 自定义插件不包含在基础 StatefulSet 中。需要额外插件时，请参考 [插件加载](configuration.md#插件加载)，通过自定义镜像、initContainer overlay、PersistentVolume 或对象存储单独管理插件。 -## 健康检查和优雅停止 +## 健康检查和停止 -Master 和 Worker 都建议添加以下容器片段。`startupProbe` 可以避免启动期误杀；`preStop` 会先调用 SeaTunnel 停止脚本，再等待 `SeaTunnelServer` 进程退出，配合 `terminationGracePeriodSeconds` 可以降低滚动更新和节点驱逐对运行任务的影响。 +Master 和 Worker 都建议添加以下探针。`startupProbe` 可以避免启动期误杀。 ```yaml startupProbe: @@ -470,6 +470,19 @@ livenessProbe: periodSeconds: 30 timeoutSeconds: 5 failureThreshold: 3 +``` + +删除 Pod 时，Kubernetes 向 PID 1 发送 SIGTERM，并在 `terminationGracePeriodSeconds` 后发送 SIGKILL。上面的 Master 和 Worker StatefulSet 已经包含干净退出所需的两项配置： + +- `command` 直接运行 `seatunnel-cluster.sh`，不通过 `/bin/sh -c` 包装。脚本会 `exec` 启动 JVM，使 JVM 成为 PID 1 并收到 SIGTERM。若 PID 1 是 `/bin/sh -c`，SIGTERM 会被忽略，Pod 要等到宽限期结束后才被强制终止。 +- `terminationGracePeriodSeconds` 让 JVM 在 SIGKILL 之前有时间执行关闭钩子。 + +收到 SIGTERM 后，Hazelcast 会按 `hazelcast.shutdownhook.policy` 执行关闭钩子，节点停止其上运行的任务并在数秒内离开集群，这些任务从最近的 checkpoint 恢复。这是快速、干净地退出集群，而不是优雅地停止运行中的作业。`GRACEFUL` 只会让 Hazelcast 在成员离开前迁移分区数据；若使用 `GRACEFUL`，请确保 `hazelcast.graceful.shutdown.max.wait` 小于 `terminationGracePeriodSeconds`。 + +:::tip 可选的 preStop +只有当容器无法直接运行脚本时（例如 `command` 必须保留 `/bin/sh -c`）才需要 `preStop`。`stop-seatunnel-cluster.sh` 会直接向 `SeaTunnelServer` 进程发送 SIGTERM，随后钩子等待该进程退出： + +```yaml lifecycle: preStop: exec: @@ -482,6 +495,7 @@ lifecycle: sleep 1 done ``` +::: ## 应用配置 diff --git a/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel-cluster.sh b/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel-cluster.sh index a18431c518..1a51667f4a 100755 --- a/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel-cluster.sh +++ b/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel-cluster.sh @@ -229,5 +229,6 @@ if [[ $DAEMON == true && $HELP == false ]]; then touch $OUT nohup java ${JAVA_OPTS} -cp ${CLASS_PATH} ${APP_MAIN} ${args} > "$OUT" 200<&- 2>&1 < /dev/null & else - java ${JAVA_OPTS} -cp ${CLASS_PATH} ${APP_MAIN} ${args} + # exec so the JVM is PID 1 in containers and receives SIGTERM directly + exec java ${JAVA_OPTS} -cp ${CLASS_PATH} ${APP_MAIN} ${args} fi diff --git a/seatunnel-core/seatunnel-starter/src/test/java/org/apache/seatunnel/core/starter/seatunnel/SeaTunnelClusterScriptTest.java b/seatunnel-core/seatunnel-starter/src/test/java/org/apache/seatunnel/core/starter/seatunnel/SeaTunnelClusterScriptTest.java index 4f6ecddfb9..ddefc0e872 100644 --- a/seatunnel-core/seatunnel-starter/src/test/java/org/apache/seatunnel/core/starter/seatunnel/SeaTunnelClusterScriptTest.java +++ b/seatunnel-core/seatunnel-starter/src/test/java/org/apache/seatunnel/core/starter/seatunnel/SeaTunnelClusterScriptTest.java @@ -140,6 +140,24 @@ public class SeaTunnelClusterScriptTest { "SEATUNNEL_HOME should use the externally configured SeaTunnel home"); } + /** + * Verifies that the foreground start replaces the shell with the JVM, so a SIGTERM sent to the + * started process reaches the JVM instead of a shell that ignores it. + */ + @Test + public void testClusterScriptExecsServerJvmInForeground() throws Exception { + Path appDirectory = createMinimalDistribution(); + runClusterScript(appDirectory, null); + + List<String> javaParentPid = + Files.readAllLines( + temporaryDirectory.resolve("java-args.txt.ppid"), StandardCharsets.UTF_8); + Assertions.assertEquals( + String.valueOf(ProcessHandle.current().pid()), + javaParentPid.get(0).trim(), + "The JVM should replace the script process (exec), not run as its child"); + } + /** * Verifies that the Windows cluster script keeps custom SeaTunnel home values and forwards the * effective value before later JVM option appends. @@ -250,6 +268,7 @@ public class SeaTunnelClusterScriptTest { Files.write( fakeJava, ("#!/bin/sh\n" + + "echo \"$PPID\" > \"$CAPTURE_FILE.ppid\"\n" + "for arg in \"$@\"; do\n" + " printf '%s\\n' \"$arg\"\n" + "done > \"$CAPTURE_FILE\"\n")
