This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/dev/pr-12693-c6a240c4666e750cd88d8e1d59cb5cc2b2134db3 in repository https://gitbox.apache.org/repos/asf/seatunnel.git
commit 016f2c153f25d1e9ce20a7250b44ab9e626a3335 Author: Jast <[email protected]> AuthorDate: Sun Oct 25 14:33:53 2026 +0000 [Docs] Fix getting-started and engine docs verified against source code (#12693) Co-authored-by: zhangshenghang <[email protected]> Co-authored-by: jast <[email protected]> Co-authored-by: Claude Code <[email protected]> Co-authored-by: multica-agent <[email protected]> --- docs/en/getting-started/docker/docker.md | 8 +- .../kubernetes/hybrid-cluster-mode.md | 2 - .../kubernetes/separated-cluster-mode.md | 4 - docs/en/getting-started/locally/deployment.md | 10 +- .../getting-started/locally/quick-start-flink.md | 7 + .../getting-started/locally/run-your-first-job.md | 2 +- .../getting-started/recipes/file-to-starrocks.md | 4 +- docs/en/getting-started/recipes/http-to-jdbc.md | 4 +- docs/en/getting-started/recipes/jdbc-to-jdbc.md | 4 +- docs/en/getting-started/recipes/jdbc-to-s3.md | 6 +- .../en/getting-started/recipes/kafka-to-iceberg.md | 2 +- docs/en/getting-started/recipes/multi-table-cdc.md | 4 +- .../getting-started/recipes/mysql-cdc-to-doris.md | 4 +- .../submit-job-to-remote-zeta-cluster.md | 141 +++++++++++++-------- docs/en/introduction/concepts/config.md | 2 +- .../concepts/gravitino-type-mapping.md | 2 - .../configuration/config-encryption-decryption.md | 2 +- docs/en/transforms/rowkind-extractor.md | 6 +- docs/en/transforms/sql-udf.md | 2 +- docs/zh/getting-started/docker/docker.md | 10 +- .../kubernetes/hybrid-cluster-mode.md | 2 - .../kubernetes/separated-cluster-mode.md | 4 - docs/zh/getting-started/locally/deployment.md | 15 ++- .../getting-started/locally/quick-start-flink.md | 7 + .../locally/quick-start-seatunnel-engine.md | 15 ++- .../getting-started/locally/run-your-first-job.md | 2 +- .../getting-started/recipes/file-to-starrocks.md | 4 +- docs/zh/getting-started/recipes/http-to-jdbc.md | 4 +- docs/zh/getting-started/recipes/jdbc-to-jdbc.md | 4 +- docs/zh/getting-started/recipes/jdbc-to-s3.md | 6 +- .../zh/getting-started/recipes/kafka-to-iceberg.md | 2 +- docs/zh/getting-started/recipes/multi-table-cdc.md | 4 +- .../getting-started/recipes/mysql-cdc-to-doris.md | 4 +- .../submit-job-to-remote-zeta-cluster.md | 136 ++++++++++++-------- docs/zh/introduction/concepts/config.md | 2 +- .../introduction/concepts/connector-v2-features.md | 4 +- .../concepts/gravitino-type-mapping.md | 2 - .../configuration/config-encryption-decryption.md | 7 +- docs/zh/introduction/configuration/sql-config.md | 2 +- docs/zh/transforms/replace.md | 2 +- docs/zh/transforms/rowkind-extractor.md | 6 +- docs/zh/transforms/sql-udf.md | 5 +- 42 files changed, 289 insertions(+), 176 deletions(-) diff --git a/docs/en/getting-started/docker/docker.md b/docs/en/getting-started/docker/docker.md index 3a32367d71..ec660195ee 100644 --- a/docs/en/getting-started/docker/docker.md +++ b/docs/en/getting-started/docker/docker.md @@ -138,8 +138,10 @@ docker run --rm -it apache/seatunnel bash ./bin/start-seatunnel-spark-3-connecto ```shell # flink version between `1.12.x` and `1.14.x` docker run --rm -it apache/seatunnel bash -c '<YOUR_FLINK_HOME>/bin/start-cluster.sh && ./bin/start-seatunnel-flink-13-connector-v2.sh -c config/v2.streaming.conf.template' -# flink version between `1.15.x` and `1.16.x` +# flink version between `1.15.x` and `1.18.x` docker run --rm -it apache/seatunnel bash -c '<YOUR_FLINK_HOME>/bin/start-cluster.sh && ./bin/start-seatunnel-flink-15-connector-v2.sh -c config/v2.streaming.conf.template' +# flink version `1.20.x` +docker run --rm -it apache/seatunnel bash -c '<YOUR_FLINK_HOME>/bin/start-cluster.sh && ./bin/start-seatunnel-flink-20-connector-v2.sh -c config/v2.streaming.conf.template' ``` @@ -239,6 +241,7 @@ services: " ports: - "5801:5801" + - "8080:8080" networks: - seatunnel_network @@ -283,7 +286,7 @@ run `docker-compose up -d` command to start the cluster. You can run `docker logs -f seatunnel_master`, `docker logs -f seatunnel_worker_1` to check the node log. -And when you call `http://localhost:5801/hazelcast/rest/maps/system-monitoring-information`, you will see there are 2 nodes as we excepted. +And when you call `http://localhost:8080/system-monitoring-information`, you will see there are 3 nodes (1 master and 2 workers) as we expected. After that, you can use client or restapi to submit job to this cluster. @@ -306,6 +309,7 @@ services: " ports: - "5801:5801" + - "8080:8080" networks: - seatunnel_network diff --git a/docs/en/getting-started/kubernetes/hybrid-cluster-mode.md b/docs/en/getting-started/kubernetes/hybrid-cluster-mode.md index 4ae2a7cabd..1155a10221 100644 --- a/docs/en/getting-started/kubernetes/hybrid-cluster-mode.md +++ b/docs/en/getting-started/kubernetes/hybrid-cluster-mode.md @@ -244,8 +244,6 @@ spec: env: - name: SEATUNNEL_HOME value: /opt/seatunnel - - name: HAZELCAST_CLUSTER_NAME - value: seatunnel-cluster ports: - containerPort: 8080 name: rest-api diff --git a/docs/en/getting-started/kubernetes/separated-cluster-mode.md b/docs/en/getting-started/kubernetes/separated-cluster-mode.md index 9a297fae87..bfe0609e60 100644 --- a/docs/en/getting-started/kubernetes/separated-cluster-mode.md +++ b/docs/en/getting-started/kubernetes/separated-cluster-mode.md @@ -334,8 +334,6 @@ spec: env: - name: SEATUNNEL_HOME value: /opt/seatunnel - - name: HAZELCAST_CLUSTER_NAME - value: seatunnel-cluster ports: - containerPort: 8080 name: rest-api @@ -407,8 +405,6 @@ spec: env: - name: SEATUNNEL_HOME value: /opt/seatunnel - - name: HAZELCAST_CLUSTER_NAME - value: seatunnel-cluster ports: - containerPort: 8080 name: rest-api diff --git a/docs/en/getting-started/locally/deployment.md b/docs/en/getting-started/locally/deployment.md index 231c0851d2..41cdabb45e 100644 --- a/docs/en/getting-started/locally/deployment.md +++ b/docs/en/getting-started/locally/deployment.md @@ -1,5 +1,5 @@ --- -sidebar_position: 2 +sidebar_position: 5 --- import Tabs from '@theme/Tabs'; @@ -27,14 +27,14 @@ wget "https://archive.apache.org/dist/seatunnel/${version}/apache-seatunnel-${ve tar -xzvf "apache-seatunnel-${version}-bin.tar.gz" ``` -On Windows, download the `.zip` archive from the [SeaTunnel Download Page](https://seatunnel.apache.org/download), then extract it with File Explorer or PowerShell: +On Windows, download the `.tar.gz` archive from the [SeaTunnel Download Page](https://seatunnel.apache.org/download), then extract it with `tar` (bundled with Windows 10 1803+). Only the `.tar.gz` archive is published on archive.apache.org, there is no `.zip` release: ```powershell $version = "3.1.0" Invoke-WebRequest ` - "https://archive.apache.org/dist/seatunnel/$version/apache-seatunnel-$version-bin.zip"; ` - -OutFile "apache-seatunnel-$version-bin.zip" -Expand-Archive "apache-seatunnel-$version-bin.zip" -DestinationPath . + "https://archive.apache.org/dist/seatunnel/$version/apache-seatunnel-$version-bin.tar.gz"; ` + -OutFile "apache-seatunnel-$version-bin.tar.gz" +tar -xzf "apache-seatunnel-$version-bin.tar.gz" Set-Location "apache-seatunnel-$version" ``` diff --git a/docs/en/getting-started/locally/quick-start-flink.md b/docs/en/getting-started/locally/quick-start-flink.md index 1cad5a151a..096b792fa6 100644 --- a/docs/en/getting-started/locally/quick-start-flink.md +++ b/docs/en/getting-started/locally/quick-start-flink.md @@ -92,6 +92,13 @@ cd "apache-seatunnel-${version}" ./bin/start-seatunnel-flink-15-connector-v2.sh --config ./config/v2.streaming.conf.template ``` +Flink version `1.20.x` + +```shell +cd "apache-seatunnel-${version}" +./bin/start-seatunnel-flink-20-connector-v2.sh --config ./config/v2.streaming.conf.template +``` + **See The Output**: When you run the command, you can see its output in your console. This is a sign to determine whether the command ran successfully or not. diff --git a/docs/en/getting-started/locally/run-your-first-job.md b/docs/en/getting-started/locally/run-your-first-job.md index c0d42defbf..39b1a01374 100644 --- a/docs/en/getting-started/locally/run-your-first-job.md +++ b/docs/en/getting-started/locally/run-your-first-job.md @@ -27,7 +27,7 @@ Install the plugins and confirm they were downloaded into `${SEATUNNEL_HOME}/con ```bash cd "${SEATUNNEL_HOME}" sh bin/install-plugin.sh -ls connectors | rg 'connector-(fake|console)' +ls connectors | grep -E 'connector-(fake|console)' ``` ## Step 3: Use a minimal job diff --git a/docs/en/getting-started/recipes/file-to-starrocks.md b/docs/en/getting-started/recipes/file-to-starrocks.md index dfa08d8fcf..4c4d7ff9de 100644 --- a/docs/en/getting-started/recipes/file-to-starrocks.md +++ b/docs/en/getting-started/recipes/file-to-starrocks.md @@ -23,13 +23,13 @@ connector-starrocks ```bash cd "${SEATUNNEL_HOME}" sh bin/install-plugin.sh -ls connectors | rg 'connector-(file-local|starrocks)' +ls connectors | grep -E 'connector-(file-local|starrocks)' ``` 3. Put the MySQL JDBC driver required by the StarRocks sink into `${SEATUNNEL_HOME}/lib`, then confirm the jar is visible: ```bash -ls "${SEATUNNEL_HOME}/lib" | rg 'mysql-connector' +ls "${SEATUNNEL_HOME}/lib" | grep -E 'mysql-connector' ``` 4. Prepare the local input file and make sure the SeaTunnel process can read it: diff --git a/docs/en/getting-started/recipes/http-to-jdbc.md b/docs/en/getting-started/recipes/http-to-jdbc.md index 9d239fa053..9ebc41741c 100644 --- a/docs/en/getting-started/recipes/http-to-jdbc.md +++ b/docs/en/getting-started/recipes/http-to-jdbc.md @@ -23,13 +23,13 @@ connector-jdbc ```bash cd "${SEATUNNEL_HOME}" sh bin/install-plugin.sh -ls connectors | rg 'connector-(http-base|jdbc)' +ls connectors | grep -E 'connector-(http-base|jdbc)' ``` 3. Put the target database JDBC driver into `${SEATUNNEL_HOME}/lib`, then confirm the jar is visible: ```bash -ls "${SEATUNNEL_HOME}/lib" | rg 'postgresql' +ls "${SEATUNNEL_HOME}/lib" | grep -E 'postgresql' ``` 4. Inspect the HTTP response before running the job. The sample endpoint from the [Http source](../../connectors/source/Http.md) should return a JSON body that contains top-level fields such as `c_string` and `c_int`: diff --git a/docs/en/getting-started/recipes/jdbc-to-jdbc.md b/docs/en/getting-started/recipes/jdbc-to-jdbc.md index d62984c8a0..cbe549537b 100644 --- a/docs/en/getting-started/recipes/jdbc-to-jdbc.md +++ b/docs/en/getting-started/recipes/jdbc-to-jdbc.md @@ -22,7 +22,7 @@ connector-jdbc ```bash cd "${SEATUNNEL_HOME}" sh bin/install-plugin.sh -ls connectors | rg 'connector-jdbc' +ls connectors | grep -E 'connector-jdbc' ``` The `Sql` transform is included in the SeaTunnel distribution and does not require another connector entry. @@ -30,7 +30,7 @@ The `Sql` transform is included in the SeaTunnel distribution and does not requi 3. Put both database drivers into `${SEATUNNEL_HOME}/lib`, then confirm that SeaTunnel can see them: ```bash -ls "${SEATUNNEL_HOME}/lib" | rg 'mysql-connector|postgresql' +ls "${SEATUNNEL_HOME}/lib" | grep -E 'mysql-connector|postgresql' ``` 4. Create the MySQL source table and insert three deterministic rows: diff --git a/docs/en/getting-started/recipes/jdbc-to-s3.md b/docs/en/getting-started/recipes/jdbc-to-s3.md index 161b2c51c5..f565d22e8b 100644 --- a/docs/en/getting-started/recipes/jdbc-to-s3.md +++ b/docs/en/getting-started/recipes/jdbc-to-s3.md @@ -23,19 +23,19 @@ connector-file-s3 ```bash cd "${SEATUNNEL_HOME}" sh bin/install-plugin.sh -ls connectors | rg 'connector-(jdbc|file-s3)' +ls connectors | grep -E 'connector-(jdbc|file-s3)' ``` 3. Put the source database JDBC driver into `${SEATUNNEL_HOME}/lib` for SeaTunnel Zeta, then confirm the jar is visible: ```bash -ls "${SEATUNNEL_HOME}/lib" | rg 'mysql-connector' +ls "${SEATUNNEL_HOME}/lib" | grep -E 'mysql-connector' ``` 4. Put `hadoop-aws` and the AWS SDK bundle required by the S3 connector into `${SEATUNNEL_HOME}/lib`, then confirm both jars are present: ```bash -ls "${SEATUNNEL_HOME}/lib" | rg 'hadoop-aws|aws-java-sdk-bundle' +ls "${SEATUNNEL_HOME}/lib" | grep -E 'hadoop-aws|aws-java-sdk-bundle' ``` 5. Prepare the source table in the relational database. This example uses MySQL and exports two rows from `analytics.orders`: diff --git a/docs/en/getting-started/recipes/kafka-to-iceberg.md b/docs/en/getting-started/recipes/kafka-to-iceberg.md index ab50c86758..c401641e3e 100644 --- a/docs/en/getting-started/recipes/kafka-to-iceberg.md +++ b/docs/en/getting-started/recipes/kafka-to-iceberg.md @@ -23,7 +23,7 @@ connector-iceberg ```bash cd "${SEATUNNEL_HOME}" sh bin/install-plugin.sh -ls connectors | rg 'connector-(kafka|iceberg)' +ls connectors | grep -E 'connector-(kafka|iceberg)' ``` 3. If you use Flink or Spark, add the Iceberg dependencies required by your environment, including `hive-exec` and `libfb303` when needed. diff --git a/docs/en/getting-started/recipes/multi-table-cdc.md b/docs/en/getting-started/recipes/multi-table-cdc.md index 9f49d3521e..c154359237 100644 --- a/docs/en/getting-started/recipes/multi-table-cdc.md +++ b/docs/en/getting-started/recipes/multi-table-cdc.md @@ -23,13 +23,13 @@ connector-jdbc ```bash cd "${SEATUNNEL_HOME}" sh bin/install-plugin.sh -ls connectors | rg 'connector-(cdc-mysql|jdbc)' +ls connectors | grep -E 'connector-(cdc-mysql|jdbc)' ``` 3. If you use SeaTunnel Zeta, place both the MySQL JDBC driver and the PostgreSQL JDBC driver into `${SEATUNNEL_HOME}/lib`, then confirm they are visible: ```bash -ls "${SEATUNNEL_HOME}/lib" | rg 'mysql-connector|postgresql' +ls "${SEATUNNEL_HOME}/lib" | grep -E 'mysql-connector|postgresql' ``` 4. Prepare the MySQL source tables. Each upstream table should have a stable primary key because this recipe routes CDC changes to downstream upsert tables automatically. diff --git a/docs/en/getting-started/recipes/mysql-cdc-to-doris.md b/docs/en/getting-started/recipes/mysql-cdc-to-doris.md index e5e2fab1b9..e559b9dcee 100644 --- a/docs/en/getting-started/recipes/mysql-cdc-to-doris.md +++ b/docs/en/getting-started/recipes/mysql-cdc-to-doris.md @@ -23,13 +23,13 @@ connector-doris ```bash cd "${SEATUNNEL_HOME}" sh bin/install-plugin.sh -ls connectors | rg 'connector-(cdc-mysql|doris)' +ls connectors | grep -E 'connector-(cdc-mysql|doris)' ``` 3. If you use SeaTunnel Zeta, download the MySQL JDBC driver and place it in `${SEATUNNEL_HOME}/lib`, then confirm the jar is visible: ```bash -ls "${SEATUNNEL_HOME}/lib" | rg 'mysql-connector' +ls "${SEATUNNEL_HOME}/lib" | grep -E 'mysql-connector' ``` If you use Flink or Spark instead of Zeta, put the same driver jar into the engine plugin directory that your runtime loads. diff --git a/docs/en/getting-started/submit-job-to-remote-zeta-cluster.md b/docs/en/getting-started/submit-job-to-remote-zeta-cluster.md index b31a763e63..b62b422436 100644 --- a/docs/en/getting-started/submit-job-to-remote-zeta-cluster.md +++ b/docs/en/getting-started/submit-job-to-remote-zeta-cluster.md @@ -58,10 +58,16 @@ See [REST API v2](../engines/zeta/rest-api-v2.md) for the complete request and r ```bash docker run -d --name seatunnel \ -p 8080:8080 \ - -e ST_DOCKER_MEMBER_COUNT=1 \ - apache/seatunnel:<version> + apache/seatunnel:<version> \ + ./bin/seatunnel-cluster.sh ``` +The published image defines no `ENTRYPOINT` or `CMD`, so the cluster startup script must be +passed as the container command (the same pattern as the [Docker quickstart](docker/docker.md)). +Without `-r`, `seatunnel-cluster.sh` starts the node with the default role `master_and_worker`, +so this single container is a working one-node cluster and no extra environment variables +are needed. + ### 3.2 Submit a job from outside the container ```bash @@ -83,7 +89,8 @@ curl -X POST http://localhost:8080/submit-job \ docker run -d --name seatunnel \ -p 8080:8080 \ -v /path/to/your/jobs:/jobs \ - apache/seatunnel:<version> + apache/seatunnel:<version> \ + ./bin/seatunnel-cluster.sh # Execute inside the container docker exec seatunnel \ @@ -100,6 +107,11 @@ inside that container process. ### 4.1 Docker Compose example +The cluster membership is passed to each container with the `ST_DOCKER_MEMBER_LIST` environment +variable (this is the only cluster-discovery environment variable the engine reads). Because the +image defines no `ENTRYPOINT`, each service also starts `seatunnel-cluster.sh` explicitly with +its role, the same as the compose file in the [Docker quickstart](docker/docker.md): + ```yaml version: "3.8" services: @@ -110,7 +122,11 @@ services: - "8080:8080" - "5801:5801" environment: - ST_DOCKER_MEMBER_COUNT: 2 + ST_DOCKER_MEMBER_LIST: seatunnel-master:5801,seatunnel-worker:5801 + entrypoint: > + /bin/sh -c " + /opt/seatunnel/bin/seatunnel-cluster.sh -r master + " networks: - st-net @@ -118,7 +134,11 @@ services: image: apache/seatunnel:<version> container_name: seatunnel-worker environment: - ST_DOCKER_MEMBER_COUNT: 2 + ST_DOCKER_MEMBER_LIST: seatunnel-master:5801,seatunnel-worker:5801 + entrypoint: > + /bin/sh -c " + /opt/seatunnel/bin/seatunnel-cluster.sh -r worker + " networks: - st-net depends_on: @@ -163,12 +183,14 @@ After deploying SeaTunnel to Kubernetes (see [Kubernetes Deployment](kubernetes/ master pod's REST port to your local machine: ```bash -# Find the master pod -kubectl get pods -n seatunnel +# Find the master pod (the Helm chart deploys master as a Deployment) +MASTER_POD=$(kubectl get pods -n seatunnel \ + -l app.kubernetes.io/name=seatunnel-master \ + -o jsonpath='{.items[0].metadata.name}') # Forward REST port kubectl port-forward -n seatunnel \ - pod/seatunnel-master-0 8080:8080 + pod/${MASTER_POD} 8080:8080 ``` In a second terminal, submit a job: @@ -179,31 +201,37 @@ curl -X POST http://localhost:8080/submit-job \ -d @job.json ``` -### 5.2 Using NodePort service (staging / production) +### 5.2 Using the master Service (default chart deployment) -If your cluster exposes the SeaTunnel master via a `NodePort` service: +The Helm chart only creates a headless Service for the master, so there is no separate REST +Service or NodePort out of the box. Forward the master Service's REST port to your local machine: ```bash -# Get the NodePort -kubectl get svc -n seatunnel seatunnel-master-rest +kubectl port-forward -n seatunnel svc/seatunnel-master 8080:8080 +``` + +In a second terminal, submit a job: -# Submit via node IP and node port -curl -X POST http://<node-ip>:<node-port>/submit-job \ +```bash +curl -X POST http://localhost:8080/submit-job \ -H "Content-Type: application/json" \ -d @job.json ``` -### 5.3 Using LoadBalancer service +### 5.3 Using an Ingress (staging / production) -```bash -LB_IP=$(kubectl get svc -n seatunnel seatunnel-master-rest \ - -o jsonpath='{.status.loadBalancer.ingress[0].ip}') +If you enable the chart's Ingress (`ingress.enabled: true`, see the +[Helm Chart Reference](kubernetes/helm.md)), submit jobs through the Ingress host instead: -curl -X POST http://${LB_IP}:8080/submit-job \ +```bash +curl -X POST http://<your-ingress-host>/submit-job \ -H "Content-Type: application/json" \ -d @job.json ``` +If you expose the REST API with your own `NodePort` or `LoadBalancer` Service, submit to that +Service's address the same way. + --- ## 6. Kubernetes: Job Configuration via ConfigMap @@ -246,10 +274,15 @@ volumes: name: seatunnel-job-config ``` -Then submit via `kubectl exec`: +Then submit via `kubectl exec` (`${MASTER_POD}` is only defined in section 5.1, so repeat the +lookup here if you jumped straight to this section): ```bash -kubectl exec -n seatunnel seatunnel-master-0 -- \ +MASTER_POD=$(kubectl get pods -n seatunnel \ + -l app.kubernetes.io/name=seatunnel-master \ + -o jsonpath='{.items[0].metadata.name}') + +kubectl exec -n seatunnel ${MASTER_POD} -- \ /opt/seatunnel/bin/seatunnel.sh \ --config /opt/seatunnel/jobs/cdc-job.conf ``` @@ -260,42 +293,54 @@ kubectl exec -n seatunnel seatunnel-master-0 -- \ ### 7.1 Helm install +The SeaTunnel Helm chart is published as an OCI artifact on Docker Hub (there is no +`https://apache.github.io/seatunnel-helm-charts` chart repository): + ```bash -helm repo add seatunnel https://apache.github.io/seatunnel-helm-charts -helm repo update +# Keep the chart version in sync with kubernetes/helm.md +export VERSION=3.1.0 +helm pull oci://registry-1.docker.io/apache/seatunnel-helm --version ${VERSION} +tar -xvf seatunnel-helm-${VERSION}.tgz +cd seatunnel-helm -helm install seatunnel seatunnel/seatunnel \ +helm install seatunnel . \ --namespace seatunnel \ - --create-namespace \ - --set master.replicaCount=2 \ - --set worker.replicaCount=4 \ - --set master.service.type=LoadBalancer + --create-namespace ``` -### 7.2 EKS-specific: retrieve the load balancer hostname +Scale masters and workers with the `master.replicas` and `worker.replicas` values: ```bash -kubectl get svc -n seatunnel seatunnel-master \ - -o jsonpath='{.status.loadBalancer.ingress[0].hostname}' +helm upgrade seatunnel . --namespace seatunnel \ + --set master.replicas=2 \ + --set worker.replicas=4 ``` -Use that hostname as the API endpoint: +### 7.2 EKS-specific: reaching the REST API + +The default chart creates only headless Services, so there is no automatically provisioned load +balancer. Either enable the chart Ingress and read its address: ```bash -export ST_HOST=$(kubectl get svc -n seatunnel seatunnel-master \ - -o jsonpath='{.status.loadBalancer.ingress[0].hostname}') +kubectl get ingress -n seatunnel +``` + +and use that host as the API endpoint: -curl -X POST http://${ST_HOST}:8080/submit-job \ +```bash +curl -X POST http://<your-ingress-host>/submit-job \ -H "Content-Type: application/json" \ -d @job.json ``` +or use `kubectl port-forward svc/seatunnel-master 8080:8080` as shown in section 5.2. + ### 7.3 Customizing resources via Helm values ```yaml # values-prod.yaml master: - replicaCount: 2 + replicas: "2" resources: requests: memory: "4Gi" @@ -305,7 +350,7 @@ master: cpu: "4" worker: - replicaCount: 8 + replicas: "8" resources: requests: memory: "8Gi" @@ -313,22 +358,18 @@ worker: limits: memory: "16Gi" cpu: "8" - -seatunnel: - config: - engine: - backup-count: 2 - queue-type: blockingqueue - print-execution-info-interval: 60 - http: - enable-http: true - port: 8080 ``` +The chart has no `seatunnel.config` values tree. To customize engine files such as +`seatunnel.yaml` or `hazelcast*.yaml`, let the chart generate the ConfigMap from the chart's +`conf/` files (default), or ship your own ConfigMap and set `configMap.create: false` plus +`configMap.existingConfigMapName` — see the chart's `values.yaml` and the +[Helm Chart Reference](kubernetes/helm.md). + Apply with: ```bash -helm upgrade seatunnel seatunnel/seatunnel \ +helm upgrade seatunnel . \ --namespace seatunnel \ -f values-prod.yaml ``` @@ -359,7 +400,7 @@ spec: - from: - namespaceSelector: matchLabels: - name: seatunnel + kubernetes.io/metadata.name: seatunnel ports: - port: 5801 ``` @@ -374,7 +415,7 @@ spec: | Workers not joining the cluster | Firewall blocks port 5801 | Open TCP 5801 between all cluster nodes | | `kubectl port-forward` disconnects | Idle timeout or pod restart | Restart port-forward; consider NodePort instead | | Job submitted but status always `WAITING` | No available worker slots | Scale up worker replicas or check resource quotas | -| EKS LoadBalancer hostname not resolving | DNS propagation delay | Wait 1–2 minutes; verify with `nslookup` | +| EKS Ingress hostname not resolving | DNS propagation delay | Wait 1–2 minutes; verify with `nslookup` | | Helm install fails with `pending-install` | Previous failed release | Run `helm rollback` or `helm uninstall` then retry | --- diff --git a/docs/en/introduction/concepts/config.md b/docs/en/introduction/concepts/config.md index 85e5db9d28..ed9471be8b 100644 --- a/docs/en/introduction/concepts/config.md +++ b/docs/en/introduction/concepts/config.md @@ -305,7 +305,7 @@ transform { sql { plugin_input = "fake_test_table" plugin_output = "sql" - query = "select * from dual where name = 'abc' " + query = "select * from fake_test_table where name = 'abc' " } } diff --git a/docs/en/introduction/concepts/gravitino-type-mapping.md b/docs/en/introduction/concepts/gravitino-type-mapping.md index bcd159ccfa..0fd73ba7eb 100644 --- a/docs/en/introduction/concepts/gravitino-type-mapping.md +++ b/docs/en/introduction/concepts/gravitino-type-mapping.md @@ -85,5 +85,3 @@ Gravitino indexes are mapped to SeaTunnel constraints: - [Gravitino Column Types](https://gravitino.apache.org/docs/1.1.0/manage-relational-metadata-using-gravitino/#apache-gravitino-table-column-type) - [Schema Feature](./schema-feature.md) -- SeaTunnel Data Types -- [SeaTunnel Data Types](./schema-feature.md) diff --git a/docs/en/introduction/configuration/config-encryption-decryption.md b/docs/en/introduction/configuration/config-encryption-decryption.md index 432b114bea..96c7267dfc 100644 --- a/docs/en/introduction/configuration/config-encryption-decryption.md +++ b/docs/en/introduction/configuration/config-encryption-decryption.md @@ -24,7 +24,7 @@ for example `access.key`, `access_key`, and `access-key` can share the same mask Next, I'll show how to quickly use SeaTunnel's own `base64` encryption: -1. And new option `shade.identifier` and `shade.options` in env block of config file, `shade.identifier` indicate what the encryption method that you want to use, while `shade.options` specifies which parameters should be encrypted/decrypted. In this example, we should add `shade.identifier = base64` in config as the following shown: +1. Add new option `shade.identifier` and `shade.options` in env block of config file, `shade.identifier` indicate what the encryption method that you want to use, while `shade.options` specifies which parameters should be encrypted/decrypted. In this example, we should add `shade.identifier = base64` in config as the following shown: ```hocon # diff --git a/docs/en/transforms/rowkind-extractor.md b/docs/en/transforms/rowkind-extractor.md index 214b215311..025284db61 100644 --- a/docs/en/transforms/rowkind-extractor.md +++ b/docs/en/transforms/rowkind-extractor.md @@ -24,11 +24,13 @@ Input (CDC data): RowKind: -D (DELETE) Data: id=1, name="test1", age=20 -Output (Append-Only data): +Output (Append-Only data, with the default `transform_type = SHORT`): RowKind: +I (INSERT) - Data: id=1, name="test1", age=20, row_kind="DELETE" + Data: id=1, name="test1", age=20, row_kind="-D" ``` +With `transform_type = FULL`, the field stores the full name instead: `row_kind="DELETE"`. + **Typical Use Cases:** - Writing CDC data to data lakes that only support Append mode - Preserving complete change history in data warehouses diff --git a/docs/en/transforms/sql-udf.md b/docs/en/transforms/sql-udf.md index bc331b7051..e37e22bc6b 100644 --- a/docs/en/transforms/sql-udf.md +++ b/docs/en/transforms/sql-udf.md @@ -11,7 +11,7 @@ Use UDF SPI to extend the SQL transform functions lib. ```java package org.apache.seatunnel.transform.sql.zeta; -public interface ZetaUDF { +public interface ZetaUDF extends Serializable { /** * Function name * diff --git a/docs/zh/getting-started/docker/docker.md b/docs/zh/getting-started/docker/docker.md index 7163f7f831..d0893e77e5 100644 --- a/docs/zh/getting-started/docker/docker.md +++ b/docs/zh/getting-started/docker/docker.md @@ -49,7 +49,7 @@ docker images | grep apache/seatunnel ##### 分步骤构建 ```shell # Build binary package from source code -mvn clean package -DskipTests -Dskip.spotless=true +sh ./mvnw clean package -DskipTests -Dskip.spotless=true # Build docker image cd seatunnel-dist @@ -138,8 +138,10 @@ docker run --rm -it apache/seatunnel bash ./bin/start-seatunnel-spark-3-connecto ```shell # flink version between `1.12.x` and `1.14.x` docker run --rm -it apache/seatunnel bash -c '<YOUR_FLINK_HOME>/bin/start-cluster.sh && ./bin/start-seatunnel-flink-13-connector-v2.sh -c config/v2.streaming.conf.template' -# flink version between `1.15.x` and `1.16.x` +# flink version between `1.15.x` and `1.18.x` docker run --rm -it apache/seatunnel bash -c '<YOUR_FLINK_HOME>/bin/start-cluster.sh && ./bin/start-seatunnel-flink-15-connector-v2.sh -c config/v2.streaming.conf.template' +# flink version `1.20.x` +docker run --rm -it apache/seatunnel bash -c '<YOUR_FLINK_HOME>/bin/start-cluster.sh && ./bin/start-seatunnel-flink-20-connector-v2.sh -c config/v2.streaming.conf.template' ``` @@ -237,6 +239,7 @@ services: " ports: - "5801:5801" + - "8080:8080" networks: - seatunnel_network @@ -280,7 +283,7 @@ networks: 启动完成后，可以运行`docker logs -f seatunnel_master`, `docker logs -f seatunnel_worker_1`来查看节点的日志 -当你访问`http://localhost:5801/hazelcast/rest/maps/system-monitoring-information` 时，可以看到集群的状态为1个master节点，2个worker节点. +当你访问`http://localhost:8080/system-monitoring-information` 时，可以看到集群的状态为1个master节点，2个worker节点. #### 集群扩容 当你需要对集群扩容, 例如需要添加一个worker节点时 @@ -299,6 +302,7 @@ services: " ports: - "5801:5801" + - "8080:8080" networks: - seatunnel_network diff --git a/docs/zh/getting-started/kubernetes/hybrid-cluster-mode.md b/docs/zh/getting-started/kubernetes/hybrid-cluster-mode.md index 38ff8eea46..bb82b6a39d 100644 --- a/docs/zh/getting-started/kubernetes/hybrid-cluster-mode.md +++ b/docs/zh/getting-started/kubernetes/hybrid-cluster-mode.md @@ -244,8 +244,6 @@ spec: env: - name: SEATUNNEL_HOME value: /opt/seatunnel - - name: HAZELCAST_CLUSTER_NAME - value: seatunnel-cluster ports: - containerPort: 8080 name: rest-api diff --git a/docs/zh/getting-started/kubernetes/separated-cluster-mode.md b/docs/zh/getting-started/kubernetes/separated-cluster-mode.md index 207b335e1d..889c6d2fb4 100644 --- a/docs/zh/getting-started/kubernetes/separated-cluster-mode.md +++ b/docs/zh/getting-started/kubernetes/separated-cluster-mode.md @@ -334,8 +334,6 @@ spec: env: - name: SEATUNNEL_HOME value: /opt/seatunnel - - name: HAZELCAST_CLUSTER_NAME - value: seatunnel-cluster ports: - containerPort: 8080 name: rest-api @@ -407,8 +405,6 @@ spec: env: - name: SEATUNNEL_HOME value: /opt/seatunnel - - name: HAZELCAST_CLUSTER_NAME - value: seatunnel-cluster ports: - containerPort: 8080 name: rest-api diff --git a/docs/zh/getting-started/locally/deployment.md b/docs/zh/getting-started/locally/deployment.md index 63a32d4fb0..ed3f628ab4 100644 --- a/docs/zh/getting-started/locally/deployment.md +++ b/docs/zh/getting-started/locally/deployment.md @@ -1,5 +1,5 @@ --- -sidebar_position: 1 +sidebar_position: 5 --- import Tabs from '@theme/Tabs'; @@ -27,6 +27,19 @@ wget "https://archive.apache.org/dist/seatunnel/${version}/apache-seatunnel-${ve tar -xzvf "apache-seatunnel-${version}-bin.tar.gz" ``` +在 Windows 上，可以从 [SeaTunnel下载页面](https://seatunnel.apache.org/download) 下载 `.tar.gz` 压缩包， +然后使用 Windows 10 1803+ 自带的 `tar` 工具解压。archive.apache.org 上只发布 `.tar.gz` 包， +没有 `.zip` 发布包： + +```powershell +$version = "3.1.0" +Invoke-WebRequest ` + "https://archive.apache.org/dist/seatunnel/$version/apache-seatunnel-$version-bin.tar.gz"; ` + -OutFile "apache-seatunnel-$version-bin.tar.gz" +tar -xzf "apache-seatunnel-$version-bin.tar.gz" +Set-Location "apache-seatunnel-$version" +``` + ### 下载连接器插件 从2.2.0-beta版本开始，二进制包不再默认提供连接器依赖，因此在第一次使用时，您需要执行以下命令来安装连接器：(当然，您也可以从 [Apache Maven Repository](https://repo.maven.apache.org/maven2/org/apache/seatunnel/) 手动下载连接器，然后将其移动至`connectors/`目录下，如果是2.3.5之前则需要放入`connectors/seatunnel`目录下)。 diff --git a/docs/zh/getting-started/locally/quick-start-flink.md b/docs/zh/getting-started/locally/quick-start-flink.md index 7a6cb6737b..2cdd8aea3c 100644 --- a/docs/zh/getting-started/locally/quick-start-flink.md +++ b/docs/zh/getting-started/locally/quick-start-flink.md @@ -92,6 +92,13 @@ cd "apache-seatunnel-${version}" ./bin/start-seatunnel-flink-15-connector-v2.sh --config ./config/v2.streaming.conf.template ``` +Flink版本`1.20.x` + +```shell +cd "apache-seatunnel-${version}" +./bin/start-seatunnel-flink-20-connector-v2.sh --config ./config/v2.streaming.conf.template +``` + **查看输出**: 当您运行该命令时，您可以在控制台中看到它的输出。您可以认为这是命令运行成功或失败的标志。 SeaTunnel控制台将会打印一些如下日志信息: diff --git a/docs/zh/getting-started/locally/quick-start-seatunnel-engine.md b/docs/zh/getting-started/locally/quick-start-seatunnel-engine.md index 9fc7cf38f2..08c6d24dca 100644 --- a/docs/zh/getting-started/locally/quick-start-seatunnel-engine.md +++ b/docs/zh/getting-started/locally/quick-start-seatunnel-engine.md @@ -36,6 +36,12 @@ connector-console sh bin/install-plugin.sh ``` +在 Windows 上，请改用批处理脚本： + +```bat +bin\install-plugin.cmd +``` + ### 步骤 1: 部署SeaTunnel及连接器 在开始前，请确保您已经按照[部署](deployment.md)中的描述下载并部署了SeaTunnel。 @@ -103,6 +109,13 @@ cd "apache-seatunnel-${version}" ``` +在 Windows 上，请在 SeaTunnel 目录下运行对应的批处理入口： + +```bat +cd apache-seatunnel-3.0.0 +bin\seatunnel.cmd --config config\v2.batch.config.template -m local +``` + **查看输出**: 当您运行该命令时，您可以在控制台中看到它的输出。您可以认为这是命令运行成功或失败的标志。 SeaTunnel控制台将会打印一些如下日志信息: @@ -220,7 +233,7 @@ Total Failed Count : 0 :::tip -如果您想优化自己的作业，请参照连接器使用文档 +如果您想优化自己的作业，请参照 [Source-MySQL](../../connectors/source/Mysql.md) 和 [Sink-Doris](../../connectors/sink/Doris.md) 的连接器使用文档。 ::: diff --git a/docs/zh/getting-started/locally/run-your-first-job.md b/docs/zh/getting-started/locally/run-your-first-job.md index 8a1b50cd8f..60331d9eb4 100644 --- a/docs/zh/getting-started/locally/run-your-first-job.md +++ b/docs/zh/getting-started/locally/run-your-first-job.md @@ -27,7 +27,7 @@ connector-console ```bash cd "${SEATUNNEL_HOME}" sh bin/install-plugin.sh -ls connectors | rg 'connector-(fake|console)' +ls connectors | grep -E 'connector-(fake|console)' ``` ## 步骤 3：使用最小可运行配置 diff --git a/docs/zh/getting-started/recipes/file-to-starrocks.md b/docs/zh/getting-started/recipes/file-to-starrocks.md index 9662c95308..7c370e2f32 100644 --- a/docs/zh/getting-started/recipes/file-to-starrocks.md +++ b/docs/zh/getting-started/recipes/file-to-starrocks.md @@ -23,13 +23,13 @@ connector-starrocks ```bash cd "${SEATUNNEL_HOME}" sh bin/install-plugin.sh -ls connectors | rg 'connector-(file-local|starrocks)' +ls connectors | grep -E 'connector-(file-local|starrocks)' ``` 3. 把 StarRocks sink 依赖的 MySQL JDBC 驱动放进 `${SEATUNNEL_HOME}/lib`，并确认 jar 已经落盘： ```bash -ls "${SEATUNNEL_HOME}/lib" | rg 'mysql-connector' +ls "${SEATUNNEL_HOME}/lib" | grep -E 'mysql-connector' ``` 4. 先准备本地输入文件，并确保 SeaTunnel 进程能读到它： diff --git a/docs/zh/getting-started/recipes/http-to-jdbc.md b/docs/zh/getting-started/recipes/http-to-jdbc.md index 5aa6873ae8..9f5fbafdac 100644 --- a/docs/zh/getting-started/recipes/http-to-jdbc.md +++ b/docs/zh/getting-started/recipes/http-to-jdbc.md @@ -23,13 +23,13 @@ connector-jdbc ```bash cd "${SEATUNNEL_HOME}" sh bin/install-plugin.sh -ls connectors | rg 'connector-(http-base|jdbc)' +ls connectors | grep -E 'connector-(http-base|jdbc)' ``` 3. 把目标数据库 JDBC 驱动放进 `${SEATUNNEL_HOME}/lib`，并确认 jar 已经落盘： ```bash -ls "${SEATUNNEL_HOME}/lib" | rg 'postgresql' +ls "${SEATUNNEL_HOME}/lib" | grep -E 'postgresql' ``` 4. 运行任务前，先看一眼 HTTP 返回内容。这里直接使用 [Http Source](../../connectors/source/Http.md) 里的示例接口，返回 JSON 顶层应该能看到 `c_string` 和 `c_int` 这些字段： diff --git a/docs/zh/getting-started/recipes/jdbc-to-jdbc.md b/docs/zh/getting-started/recipes/jdbc-to-jdbc.md index 1b9fd11874..12c5cd44e3 100644 --- a/docs/zh/getting-started/recipes/jdbc-to-jdbc.md +++ b/docs/zh/getting-started/recipes/jdbc-to-jdbc.md @@ -22,7 +22,7 @@ connector-jdbc ```bash cd "${SEATUNNEL_HOME}" sh bin/install-plugin.sh -ls connectors | rg 'connector-jdbc' +ls connectors | grep -E 'connector-jdbc' ``` `Sql` transform 已包含在 SeaTunnel 发布包中，不需要再添加 connector 配置。 @@ -30,7 +30,7 @@ ls connectors | rg 'connector-jdbc' 3. 把两个数据库的驱动都放入 `${SEATUNNEL_HOME}/lib`，然后确认 SeaTunnel 可以看到它们： ```bash -ls "${SEATUNNEL_HOME}/lib" | rg 'mysql-connector|postgresql' +ls "${SEATUNNEL_HOME}/lib" | grep -E 'mysql-connector|postgresql' ``` 4. 创建 MySQL 源表，并插入三条固定数据： diff --git a/docs/zh/getting-started/recipes/jdbc-to-s3.md b/docs/zh/getting-started/recipes/jdbc-to-s3.md index 2ea7a42e79..0945280250 100644 --- a/docs/zh/getting-started/recipes/jdbc-to-s3.md +++ b/docs/zh/getting-started/recipes/jdbc-to-s3.md @@ -23,19 +23,19 @@ connector-file-s3 ```bash cd "${SEATUNNEL_HOME}" sh bin/install-plugin.sh -ls connectors | rg 'connector-(jdbc|file-s3)' +ls connectors | grep -E 'connector-(jdbc|file-s3)' ``` 3. 把源端数据库 JDBC 驱动放进 `${SEATUNNEL_HOME}/lib`，并确认 jar 已经落盘： ```bash -ls "${SEATUNNEL_HOME}/lib" | rg 'mysql-connector' +ls "${SEATUNNEL_HOME}/lib" | grep -E 'mysql-connector' ``` 4. 把 S3 连接器依赖的 `hadoop-aws` 和 AWS SDK bundle 也放进 `${SEATUNNEL_HOME}/lib`，然后确认这两个依赖都能看到： ```bash -ls "${SEATUNNEL_HOME}/lib" | rg 'hadoop-aws|aws-java-sdk-bundle' +ls "${SEATUNNEL_HOME}/lib" | grep -E 'hadoop-aws|aws-java-sdk-bundle' ``` 5. 先准备源端数据库表。这个示例使用 MySQL，并从 `analytics.orders` 导出两条数据： diff --git a/docs/zh/getting-started/recipes/kafka-to-iceberg.md b/docs/zh/getting-started/recipes/kafka-to-iceberg.md index bb056feba8..0d205daa24 100644 --- a/docs/zh/getting-started/recipes/kafka-to-iceberg.md +++ b/docs/zh/getting-started/recipes/kafka-to-iceberg.md @@ -23,7 +23,7 @@ connector-iceberg ```bash cd "${SEATUNNEL_HOME}" sh bin/install-plugin.sh -ls connectors | rg 'connector-(kafka|iceberg)' +ls connectors | grep -E 'connector-(kafka|iceberg)' ``` 3. 如果你使用 Flink 或 Spark，请补齐 Iceberg 在对应环境里需要的依赖，例如 `hive-exec` 和 `libfb303`。 diff --git a/docs/zh/getting-started/recipes/multi-table-cdc.md b/docs/zh/getting-started/recipes/multi-table-cdc.md index 0a5e30c1cc..173ca7c19f 100644 --- a/docs/zh/getting-started/recipes/multi-table-cdc.md +++ b/docs/zh/getting-started/recipes/multi-table-cdc.md @@ -23,13 +23,13 @@ connector-jdbc ```bash cd "${SEATUNNEL_HOME}" sh bin/install-plugin.sh -ls connectors | rg 'connector-(cdc-mysql|jdbc)' +ls connectors | grep -E 'connector-(cdc-mysql|jdbc)' ``` 3. 如果你用的是 SeaTunnel Zeta，再把 MySQL JDBC 驱动和 PostgreSQL JDBC 驱动都放进 `${SEATUNNEL_HOME}/lib`，并确认 jar 已经能看到： ```bash -ls "${SEATUNNEL_HOME}/lib" | rg 'mysql-connector|postgresql' +ls "${SEATUNNEL_HOME}/lib" | grep -E 'mysql-connector|postgresql' ``` 4. 先准备 MySQL 源表。因为这条链路会自动把 CDC 事件路由到下游 upsert 表，所以每张上游表都要有稳定主键。 diff --git a/docs/zh/getting-started/recipes/mysql-cdc-to-doris.md b/docs/zh/getting-started/recipes/mysql-cdc-to-doris.md index c3f0f32d6b..e185aee5d0 100644 --- a/docs/zh/getting-started/recipes/mysql-cdc-to-doris.md +++ b/docs/zh/getting-started/recipes/mysql-cdc-to-doris.md @@ -23,13 +23,13 @@ connector-doris ```bash cd "${SEATUNNEL_HOME}" sh bin/install-plugin.sh -ls connectors | rg 'connector-(cdc-mysql|doris)' +ls connectors | grep -E 'connector-(cdc-mysql|doris)' ``` 3. 如果你用的是 SeaTunnel Zeta，再把 MySQL JDBC 驱动放进 `${SEATUNNEL_HOME}/lib`，并确认 jar 已经落盘： ```bash -ls "${SEATUNNEL_HOME}/lib" | rg 'mysql-connector' +ls "${SEATUNNEL_HOME}/lib" | grep -E 'mysql-connector' ``` 如果你用的是 Flink 或 Spark，就把同一个驱动 jar 放到对应引擎实际加载的插件目录里。 diff --git a/docs/zh/getting-started/submit-job-to-remote-zeta-cluster.md b/docs/zh/getting-started/submit-job-to-remote-zeta-cluster.md index 3a3b3001ff..659c0191db 100644 --- a/docs/zh/getting-started/submit-job-to-remote-zeta-cluster.md +++ b/docs/zh/getting-started/submit-job-to-remote-zeta-cluster.md @@ -56,10 +56,15 @@ curl -X POST http://192.168.1.100:8080/submit-job \ ```bash docker run -d --name seatunnel \ -p 8080:8080 \ - -e ST_DOCKER_MEMBER_COUNT=1 \ - apache/seatunnel:<version> + apache/seatunnel:<version> \ + ./bin/seatunnel-cluster.sh ``` +官方镜像没有定义 `ENTRYPOINT` 或 `CMD`，因此必须把集群启动脚本作为容器命令显式传入 +（与 [Docker 快速开始](docker/docker.md) 中的写法一致）。不带 `-r` 参数时， +`seatunnel-cluster.sh` 默认以 `master_and_worker` 角色启动节点，因此这个单容器就是 +一个可用的单节点集群，无需额外环境变量。 + ### 3.2 从容器外部提交作业 ```bash @@ -81,7 +86,8 @@ curl -X POST http://localhost:8080/submit-job \ docker run -d --name seatunnel \ -p 8080:8080 \ -v /path/to/your/jobs:/jobs \ - apache/seatunnel:<version> + apache/seatunnel:<version> \ + ./bin/seatunnel-cluster.sh # 在容器内执行 docker exec seatunnel \ @@ -97,6 +103,10 @@ docker exec seatunnel \ ### 4.1 Docker Compose 示例 +集群成员列表通过 `ST_DOCKER_MEMBER_LIST` 环境变量传给每个容器（这是引擎唯一读取的集群发现环境变量）。 +由于镜像没有定义 `ENTRYPOINT`，每个服务还需要显式以各自角色启动 `seatunnel-cluster.sh`， +与 [Docker 快速开始](docker/docker.md) 中的 compose 文件一致： + ```yaml version: "3.8" services: @@ -107,7 +117,11 @@ services: - "8080:8080" - "5801:5801" environment: - ST_DOCKER_MEMBER_COUNT: 2 + ST_DOCKER_MEMBER_LIST: seatunnel-master:5801,seatunnel-worker:5801 + entrypoint: > + /bin/sh -c " + /opt/seatunnel/bin/seatunnel-cluster.sh -r master + " networks: - st-net @@ -115,7 +129,11 @@ services: image: apache/seatunnel:<version> container_name: seatunnel-worker environment: - ST_DOCKER_MEMBER_COUNT: 2 + ST_DOCKER_MEMBER_LIST: seatunnel-master:5801,seatunnel-worker:5801 + entrypoint: > + /bin/sh -c " + /opt/seatunnel/bin/seatunnel-cluster.sh -r worker + " networks: - st-net depends_on: @@ -159,12 +177,14 @@ curl -X POST http://localhost:8080/submit-job \ Pod 的 REST 端口转发到本地： ```bash -# 查找 master pod -kubectl get pods -n seatunnel +# 查找 master pod（Helm chart 以 Deployment 方式部署 master） +MASTER_POD=$(kubectl get pods -n seatunnel \ + -l app.kubernetes.io/name=seatunnel-master \ + -o jsonpath='{.items[0].metadata.name}') # 转发 REST 端口 kubectl port-forward -n seatunnel \ - pod/seatunnel-master-0 8080:8080 + pod/${MASTER_POD} 8080:8080 ``` 在另一个终端提交作业： @@ -175,31 +195,37 @@ curl -X POST http://localhost:8080/submit-job \ -d @job.json ``` -### 5.2 使用 NodePort 服务（测试 / 生产） +### 5.2 通过 master Service 提交（默认 chart 部署） -若集群通过 `NodePort` 服务暴露 Master： +Helm chart 默认只为 master 创建 headless Service，不会额外创建 REST Service 或 NodePort。 +将 master Service 的 REST 端口转发到本地： ```bash -# 获取 NodePort -kubectl get svc -n seatunnel seatunnel-master-rest +kubectl port-forward -n seatunnel svc/seatunnel-master 8080:8080 +``` + +在另一个终端提交作业： -# 使用节点 IP 和节点端口提交 -curl -X POST http://<node-ip>:<node-port>/submit-job \ +```bash +curl -X POST http://localhost:8080/submit-job \ -H "Content-Type: application/json" \ -d @job.json ``` -### 5.3 使用 LoadBalancer 服务 +### 5.3 通过 Ingress 提交（测试 / 生产） -```bash -LB_IP=$(kubectl get svc -n seatunnel seatunnel-master-rest \ - -o jsonpath='{.status.loadBalancer.ingress[0].ip}') +如果启用了 chart 的 Ingress（`ingress.enabled: true`，参见 [Helm Chart 参考](kubernetes/helm.md)）， +可以直接通过 Ingress 主机提交作业： -curl -X POST http://${LB_IP}:8080/submit-job \ +```bash +curl -X POST http://<your-ingress-host>/submit-job \ -H "Content-Type: application/json" \ -d @job.json ``` +如果你自行创建了 `NodePort` 或 `LoadBalancer` 类型的 Service 来暴露 REST API，按同样的方式向该 +Service 地址提交即可。 + --- ## 6. Kubernetes：通过 ConfigMap 管理作业配置 @@ -241,10 +267,14 @@ volumes: name: seatunnel-job-config ``` -通过 `kubectl exec` 提交： +通过 `kubectl exec` 提交（`${MASTER_POD}` 只在 5.1 节定义过，如果直接从本节开始，请先重复一次查找）： ```bash -kubectl exec -n seatunnel seatunnel-master-0 -- \ +MASTER_POD=$(kubectl get pods -n seatunnel \ + -l app.kubernetes.io/name=seatunnel-master \ + -o jsonpath='{.items[0].metadata.name}') + +kubectl exec -n seatunnel ${MASTER_POD} -- \ /opt/seatunnel/bin/seatunnel.sh \ --config /opt/seatunnel/jobs/cdc-job.conf ``` @@ -255,42 +285,53 @@ kubectl exec -n seatunnel seatunnel-master-0 -- \ ### 7.1 Helm 安装 +SeaTunnel 的 Helm chart 以 OCI 构件形式发布在 Docker Hub 上（不存在 +`https://apache.github.io/seatunnel-helm-charts` 这个 chart 仓库）： + ```bash -helm repo add seatunnel https://apache.github.io/seatunnel-helm-charts -helm repo update +# 版本号请与 kubernetes/helm.md 保持一致 +export VERSION=3.1.0 +helm pull oci://registry-1.docker.io/apache/seatunnel-helm --version ${VERSION} +tar -xvf seatunnel-helm-${VERSION}.tgz +cd seatunnel-helm -helm install seatunnel seatunnel/seatunnel \ +helm install seatunnel . \ --namespace seatunnel \ - --create-namespace \ - --set master.replicaCount=2 \ - --set worker.replicaCount=4 \ - --set master.service.type=LoadBalancer + --create-namespace ``` -### 7.2 EKS 获取 Load Balancer 主机名 +使用 `master.replicas` 和 `worker.replicas` 调整 master / worker 副本数： ```bash -kubectl get svc -n seatunnel seatunnel-master \ - -o jsonpath='{.status.loadBalancer.ingress[0].hostname}' +helm upgrade seatunnel . --namespace seatunnel \ + --set master.replicas=2 \ + --set worker.replicas=4 ``` -以该主机名作为 API 端点： +### 7.2 EKS 访问 REST API + +默认 chart 只创建 headless Service，不会自动创建负载均衡器。可以启用 chart 的 Ingress 后读取其地址： ```bash -export ST_HOST=$(kubectl get svc -n seatunnel seatunnel-master \ - -o jsonpath='{.status.loadBalancer.ingress[0].hostname}') +kubectl get ingress -n seatunnel +``` + +以该主机作为 API 端点： -curl -X POST http://${ST_HOST}:8080/submit-job \ +```bash +curl -X POST http://<your-ingress-host>/submit-job \ -H "Content-Type: application/json" \ -d @job.json ``` +或者按照 5.2 节的方式使用 `kubectl port-forward svc/seatunnel-master 8080:8080`。 + ### 7.3 通过 Helm values 自定义资源配置 ```yaml # values-prod.yaml master: - replicaCount: 2 + replicas: "2" resources: requests: memory: "4Gi" @@ -300,7 +341,7 @@ master: cpu: "4" worker: - replicaCount: 8 + replicas: "8" resources: requests: memory: "8Gi" @@ -308,22 +349,17 @@ worker: limits: memory: "16Gi" cpu: "8" - -seatunnel: - config: - engine: - backup-count: 2 - queue-type: blockingqueue - print-execution-info-interval: 60 - http: - enable-http: true - port: 8080 ``` +chart 不存在 `seatunnel.config` 这个 values 配置树。若要自定义 `seatunnel.yaml`、`hazelcast*.yaml` +等引擎配置文件，可以使用 chart 自带 `conf/` 文件生成的 ConfigMap（默认行为），或自行创建 ConfigMap +并设置 `configMap.create: false` 与 `configMap.existingConfigMapName` —— 参见 chart 的 +`values.yaml` 和 [Helm Chart 参考](kubernetes/helm.md)。 + 应用配置： ```bash -helm upgrade seatunnel seatunnel/seatunnel \ +helm upgrade seatunnel . \ --namespace seatunnel \ -f values-prod.yaml ``` @@ -354,7 +390,7 @@ spec: - from: - namespaceSelector: matchLabels: - name: seatunnel + kubernetes.io/metadata.name: seatunnel ports: - port: 5801 ``` @@ -369,7 +405,7 @@ spec: | Worker 无法加入集群 | 防火墙阻断 5801 端口 | 开放所有集群节点间 TCP 5801 | | `kubectl port-forward` 断开 | 空闲超时或 Pod 重启 | 重新执行 port-forward；考虑改用 NodePort | | 作业已提交但状态始终为 `WAITING` | 无可用 Worker 槽位 | 扩容 Worker 副本数或检查资源配额 | -| EKS LoadBalancer 主机名无法解析 | DNS 传播延迟 | 等待 1–2 分钟；用 `nslookup` 验证 | +| EKS Ingress 主机名无法解析 | DNS 传播延迟 | 等待 1–2 分钟；用 `nslookup` 验证 | | Helm 安装卡在 `pending-install` | 上次安装失败残留 | 执行 `helm rollback` 或 `helm uninstall` 后重试 | --- diff --git a/docs/zh/introduction/concepts/config.md b/docs/zh/introduction/concepts/config.md index 627a42753f..52ffb947f3 100644 --- a/docs/zh/introduction/concepts/config.md +++ b/docs/zh/introduction/concepts/config.md @@ -306,7 +306,7 @@ transform { sql { plugin_input = "fake_test_table" plugin_output = "sql" - query = "select * from dual where name = 'abc' " + query = "select * from fake_test_table where name = 'abc' " } } diff --git a/docs/zh/introduction/concepts/connector-v2-features.md b/docs/zh/introduction/concepts/connector-v2-features.md index d0894d3bfb..a67d59a35c 100644 --- a/docs/zh/introduction/concepts/connector-v2-features.md +++ b/docs/zh/introduction/concepts/connector-v2-features.md @@ -68,13 +68,13 @@ Sink connector有一些公共的核心特性，每个sink connector在不同程 对于sink connector，如果任何数据只写入目标一次，则sink connector支持精确一次。 通常有两种方法可以实现这一目标： * 目标数据库支持key去重。例如 `MySQL`, `Kudu`。 -* 目标支持 **XA 事务**(事务可以跨会话使用，即使创建事务的程序已经结束，新启动的程序也只需要知道最后一个事务的ID就可以重新提交或回滚事务）。 然后我们可以使用 **两阶段提交** 来确保 * 精确一次**。 例如：`File`, `MySQL`. +* 目标支持 **XA 事务**(事务可以跨会话使用，即使创建事务的程序已经结束，新启动的程序也只需要知道最后一个事务的ID就可以重新提交或回滚事务）。 然后我们可以使用 **两阶段提交** 来确保 **精确一次**。 例如：`File`, `MySQL`. ### cdc(更改数据捕获，change data capture) 如果sink connector支持基于主键写入行类型（INSERT/UPDATE_BEFORE/UPDATE_AFTER/DELETE），我们认为它支持cdc（更改数据捕获，change data capture）。 -### 支持多表读取 +### 支持多表写入 支持在一个 SeaTunnel 作业中写入多个表，用户可以通过[配置占位符](../configuration/sink-options-placeholders.md)动态指定表的标识符。 diff --git a/docs/zh/introduction/concepts/gravitino-type-mapping.md b/docs/zh/introduction/concepts/gravitino-type-mapping.md index 43f3f3a289..e03dbd8f61 100644 --- a/docs/zh/introduction/concepts/gravitino-type-mapping.md +++ b/docs/zh/introduction/concepts/gravitino-type-mapping.md @@ -85,5 +85,3 @@ Gravitino 索引映射到 SeaTunnel 约束： - [Gravitino 列类型](https://gravitino.apache.org/docs/1.1.0/manage-relational-metadata-using-gravitino/#apache-gravitino-table-column-type) - [Schema 特性](./schema-feature.md) -- SeaTunnel 数据类型 -- [SeaTunnel 数据类型](./schema-feature.md) diff --git a/docs/zh/introduction/configuration/config-encryption-decryption.md b/docs/zh/introduction/configuration/config-encryption-decryption.md index 6a1f5d649c..594ab0820c 100644 --- a/docs/zh/introduction/configuration/config-encryption-decryption.md +++ b/docs/zh/introduction/configuration/config-encryption-decryption.md @@ -23,8 +23,7 @@ Base64编码默认支持加密以下参数： 接下来，将展示如何快速使用 SeaTunnel 自带的 `base64` 加密功能： -1. 在配置文件的环境变量（env）部分新增了选项 `shade.identifier` 和 `shade.options`。`shade.identifier`用于表示您想要使用的加密方法，`shade.options`用于指定您想加解密的参数。 - 2. 在这个示例中，我们在配置文件中添加了 `shade.identifier = base64`，如下所示： +1. 在配置文件的环境变量（env）部分新增了选项 `shade.identifier` 和 `shade.options`。`shade.identifier`用于表示您想要使用的加密方法，`shade.options`用于指定您想加解密的参数。在这个示例中，我们在配置文件中添加了 `shade.identifier = base64`，如下所示： ```hocon # @@ -87,7 +86,7 @@ Base64编码默认支持加密以下参数： } } ``` -3. 通过Shell脚本调用不同的计算引擎来对配置文件进行加密操作。在本示例中，我们使用 Zeta 引擎对配置文件进行加密。 +2. 通过Shell脚本调用不同的计算引擎来对配置文件进行加密操作。在本示例中，我们使用 Zeta 引擎对配置文件进行加密。 ```shell ${SEATUNNEL_HOME}/bin/seatunnel.sh --config config/v2.batch.template --encrypt @@ -135,7 +134,7 @@ Base64编码默认支持加密以下参数： ] } ``` -4. 当然，不仅支持加密配置文件，还支持对配置文件的解密。如果用户想要查看解密后的配置文件，可以执行以下命令： +3. 当然，不仅支持加密配置文件，还支持对配置文件的解密。如果用户想要查看解密后的配置文件，可以执行以下命令： ```shell ${SEATUNNEL_HOME}/bin/seatunnel.sh --config config/v2.batch.template --decrypt diff --git a/docs/zh/introduction/configuration/sql-config.md b/docs/zh/introduction/configuration/sql-config.md index c6b50baf73..4f9e79d3aa 100644 --- a/docs/zh/introduction/configuration/sql-config.md +++ b/docs/zh/introduction/configuration/sql-config.md @@ -136,7 +136,7 @@ INSERT INTO sink_table SELECT source_table; ``` * `SELECT` 部分直接使用源端映射表的表名，表示将源端表的所有数据插入到目标端表中 -* 使用该语法不会生成`trasform`的相关配置，这种语法一般用在多表同步的场景，示例： +* 使用该语法不会生成`transform`的相关配置，这种语法一般用在多表同步的场景，示例： ```sql CREATE TABLE source_table WITH ( diff --git a/docs/zh/transforms/replace.md b/docs/zh/transforms/replace.md index cdefbff965..b1222f04ed 100644 --- a/docs/zh/transforms/replace.md +++ b/docs/zh/transforms/replace.md @@ -55,7 +55,7 @@ | Kin Dom | 20 | 123 | | Joy Dom | 20 | 123 | -我们想要将 `name` 字段中的字符 ``替换为 `_`。然后我们可以添加一个 `Replace` 转换，像这样： +我们想要将 `name` 字段中的字符 ` `(空格)替换为 `_`。然后我们可以添加一个 `Replace` 转换，像这样： ``` transform { diff --git a/docs/zh/transforms/rowkind-extractor.md b/docs/zh/transforms/rowkind-extractor.md index 5fb90b03cd..c0622f03e2 100644 --- a/docs/zh/transforms/rowkind-extractor.md +++ b/docs/zh/transforms/rowkind-extractor.md @@ -24,11 +24,13 @@ RowKindExtractor 转换插件用于将 CDC（Change Data Capture）数据流改 RowKind: -D (DELETE) 数据: id=1, name="test1", age=20 -输出（Append-Only 数据）： +输出（Append-Only 数据，默认 `transform_type = SHORT`）： RowKind: +I (INSERT) - 数据: id=1, name="test1", age=20, row_kind="DELETE" + 数据: id=1, name="test1", age=20, row_kind="-D" ``` +若设置 `transform_type = FULL`，该字段保存完整名称：`row_kind="DELETE"`。 + **典型应用场景：** - 将 CDC 数据写入只支持 Append 的数据湖 - 需要在数据仓库中保留完整的变更历史记录 diff --git a/docs/zh/transforms/sql-udf.md b/docs/zh/transforms/sql-udf.md index ee6c56a159..a2363b555a 100644 --- a/docs/zh/transforms/sql-udf.md +++ b/docs/zh/transforms/sql-udf.md @@ -11,7 +11,7 @@ ```java package org.apache.seatunnel.transform.sql.zeta; -public interface ZetaUDF { +public interface ZetaUDF extends Serializable { /** * Function name * @@ -130,7 +130,8 @@ public class ExampleUDF implements ZetaUDF { } ``` -打包UDF项目并将jar文件复制到路径：${SEATUNNEL_HOME}/lib +打包UDF项目并将jar文件复制到路径：`${SEATUNNEL_HOME}/lib`。如果您的 UDF 使用了第三方库，也需要将其放入 `${SEATUNNEL_HOME}/lib`。 +如果使用集群模式，您需要将相关 jar 放到所有节点的 `${SEATUNNEL_HOME}/lib` 目录下，并重启集群。 ## 支持上下文与生命周期的 UDF 示例
