This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 6d3b743b1015 test(spark): add Spark Connect SDP integration harness
and CI coverage (#20453)
6d3b743b1015 is described below
commit 6d3b743b10151272569ea9f01558a41794885df4
Author: Danny Chan <[email protected]>
AuthorDate: Mon Nov 16 06:00:01 2026 +0000
test(spark): add Spark Connect SDP integration harness and CI coverage
(#20453)
* test(spark): add Spark Connect SDP integration harness and CI coverage
* test(spark): split SDP CLI coverage between Maven and nightly
* test(spark): run SDP checks in existing CI jobs
---
.github/workflows/bot.yml | 70 ++++++
hudi-integ-test/spark-pipelines/README.md | 172 ++++++++++++++
hudi-integ-test/spark-pipelines/client.py | 59 +++++
hudi-integ-test/spark-pipelines/harness.py | 251 +++++++++++++++++++++
hudi-integ-test/spark-pipelines/pipeline.py | 28 +++
hudi-integ-test/spark-pipelines/requirements.txt | 18 ++
hudi-integ-test/spark-pipelines/test_harness.py | 98 ++++++++
hudi-spark-datasource/hudi-spark/pom.xml | 48 ++++
.../hudi/TestHoodieSparkConnectPipelines.scala | 218 ++++++++++++++++++
9 files changed, 962 insertions(+)
diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml
index 923d0056e812..dcdf98a7cdfa 100644
--- a/.github/workflows/bot.yml
+++ b/.github/workflows/bot.yml
@@ -955,6 +955,42 @@ jobs:
with:
flags: spark-scala-tests
token: ${{ secrets.CODECOV_TOKEN }}
+ # Run SDP only in the existing Spark 4.2 matrix entry, after its tests
and
+ # coverage upload. A clean Spark 4.1 rebuild prevents mixed-profile
classes.
+ - name: Set up Python for SDP
+ if: matrix.sparkProfile == 'spark4.2'
+ uses: actions/setup-python@v5
+ with:
+ python-version: '3.11'
+ cache: pip
+ cache-dependency-path:
hudi-integ-test/spark-pipelines/requirements.txt
+ - name: Install pinned SDP CLI dependencies
+ if: matrix.sparkProfile == 'spark4.2'
+ run: python3 -m pip install -r
hudi-integ-test/spark-pipelines/requirements.txt
+ - name: Build Spark 4.1 bundle and run SDP process tests
+ if: matrix.sparkProfile == 'spark4.2'
+ timeout-minutes: 25
+ run: |
+ python3 -B -m unittest discover -s hudi-integ-test/spark-pipelines
-p 'test_harness.py' -v
+ mvn clean package -T 2 -Pjava17 -Dspark4.1 -Dscala-2.13 -DskipTests \
+ $MVN_ARGS -am -pl packaging/hudi-spark-bundle
+ shopt -s nullglob
+
bundles=(packaging/hudi-spark-bundle/target/hudi-spark4.1-bundle_2.13-*.jar)
+ candidates=()
+ for bundle in "${bundles[@]}"; do
+ [[ "$bundle" == *-sources.jar || "$bundle" == *-javadoc.jar ]] ||
candidates+=("$bundle")
+ done
+ [[ ${#candidates[@]} -eq 1 ]] || { echo "Expected one candidate
bundle, found ${#candidates[@]}"; exit 1; }
+ python3 hudi-integ-test/spark-pipelines/harness.py \
+ --bundle "${candidates[0]}" --logs "$RUNNER_TEMP/sdp-logs"
+ - name: Upload SDP process diagnostics
+ if: always() && matrix.sparkProfile == 'spark4.2'
+ uses: actions/upload-artifact@v4
+ with:
+ name: spark-connect-layer3-logs
+ path: ${{ runner.temp }}/sdp-logs
+ if-no-files-found: ignore
+
test-spark-java17-scala-other-tests:
if: *route
@@ -1016,6 +1052,40 @@ jobs:
with:
flags: spark-scala-tests
token: ${{ secrets.CODECOV_TOKEN }}
+ # Run SDP only in the existing Spark 4.2 matrix entry, after its tests
and
+ # coverage upload. A clean Spark 4.1 rebuild prevents mixed-profile
classes.
+ - name: Set up Python for SDP
+ if: matrix.sparkProfile == 'spark4.2'
+ uses: actions/setup-python@v5
+ with:
+ python-version: '3.11'
+ cache: pip
+ cache-dependency-path:
hudi-integ-test/spark-pipelines/requirements.txt
+ - name: Install pinned SDP CLI dependencies
+ if: matrix.sparkProfile == 'spark4.2'
+ run: python3 -m pip install -r
hudi-integ-test/spark-pipelines/requirements.txt
+ - name: Build Spark 4.1 dependencies and run SDP ScalaTests
+ if: matrix.sparkProfile == 'spark4.2'
+ timeout-minutes: 25
+ env:
+ HUDI_SDP_REQUIRE_PYTHON: 'true'
+ run: |
+ mvn clean install -T 2 -Pjava17 -Dspark4.1 -Dscala-2.13 -DskipTests \
+ $MVN_ARGS -am -pl hudi-spark-datasource/hudi-spark
+ mvn test -Punit-tests -Pjava17 -Dspark4.1 -Dscala-2.13 \
+ -Dtest=skipJavaTests -Dsurefire.failIfNoSpecifiedTests=false \
+
-Dsuites=org.apache.spark.sql.connect.hudi.TestHoodieSparkConnectPipelines \
+ $MVN_ARGS -pl hudi-spark-datasource/hudi-spark
+ - name: Upload SDP ScalaTest diagnostics
+ if: always() && matrix.sparkProfile == 'spark4.2'
+ uses: actions/upload-artifact@v4
+ with:
+ name: spark-connect-layer2-logs
+ path: |
+ hudi-spark-datasource/hudi-spark/target/sdp-layer2
+ hudi-spark-datasource/hudi-spark/target/surefire-reports
+ if-no-files-found: ignore
+
# [CI-TRIM] to revisit for CI improvement
# build-spark-java17:
diff --git a/hudi-integ-test/spark-pipelines/README.md
b/hudi-integ-test/spark-pipelines/README.md
new file mode 100644
index 000000000000..00c65567cba1
--- /dev/null
+++ b/hudi-integ-test/spark-pipelines/README.md
@@ -0,0 +1,172 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Spark Connect / Declarative Pipelines integration harness
+
+The process harness runs the **real `spark-pipelines` CLI** against a
separately launched
+Spark 4.1.1 Connect JVM with the candidate Hudi bundle. It supports Linux and
+macOS, JDK 17, and Python 3.10 or newer (CI uses Python 3.11).
+The server and clients share a local filesystem; no external storage or
metastore
+is required. Use a dedicated Python virtual environment.
+
+## Two test layers
+
+Layer 2 invokes `python -m pyspark.pipelines.cli` with `SPARK_REMOTE`; layer 3
+also exercises the `spark-pipelines` launcher. Both use PySpark 4.1.1. Neither
substitutes
+in-JVM `DataflowGraph` calls for the client/server protocol.
+
+| Layer | Server | Coverage | Execution |
+| --- | --- | --- | --- |
+| 2: `TestHoodieSparkConnectPipelines` | Connect service in the ScalaTest JVM,
Hudi on the Maven test classpath | MV/ST exact rows, independent SQL flow
insert/upsert settings, null-key failure and a fresh remote query afterward |
Existing Scala-other job on PRs, pushes and merge groups |
+| 3: `harness.py` | Separate `spark-submit` JVM with the candidate bundle |
Bundle packaging, static server configuration, remote writes/reads, process
ownership and cleanup | Existing Scala-DML job on PRs, pushes and merge groups |
+
+Both layers run as additional steps in existing `bot.yml` jobs; no new
workflow,
+job, or matrix entry is added. Layer 2 runs in
`test-spark-java17-scala-other-tests`
+and layer 3 in `test-spark-java17-scala-dml-tests`, only in their existing
Spark 4.2
+matrix entries. Each first completes its normal tests and coverage upload, then
+performs a clean Spark 4.1 rebuild so compiled classes cannot leak across
profiles.
+Both jobs already feed `Java CI result`; an SDP failure therefore fails normal
PR
+validation. The separate nightly workflow and its retry policy are removed.
+
+The November 11 guidance from Ethan and Siva permits every-PR execution with
less
+than 30 minutes of additional runtime per job. Python setup/install, the Spark
+4.1 build and the tests must be included in that measurement. The build/test
step
+has a 25-minute timeout to leave headroom; a timeout is a failure, not evidence
+that the performance target was met. CI timings must confirm the complete added
+cost independently for each job. There is no combined runtime budget across the
+two jobs and no new global nightly gate or bypass mechanism.
+
+### Run the layer-2 suite locally
+
+With JDK 17, install the pinned `requirements.txt` in a Python environment,
then:
+
+```sh
+mvn install -T 2 -Pjava17 -Dspark4.1 -Dscala-2.13 -DskipTests \
+ -Dgpg.skip -Djacoco.skip -am -pl hudi-spark-datasource/hudi-spark
+HUDI_SDP_PYTHON=/path/to/venv/bin/python HUDI_SDP_REQUIRE_PYTHON=true \
+ mvn test -Punit-tests -Pjava17 -Dspark4.1 -Dscala-2.13 \
+ -Dtest=skipJavaTests -Dsurefire.failIfNoSpecifiedTests=false \
+ -Dsuites=org.apache.spark.sql.connect.hudi.TestHoodieSparkConnectPipelines \
+ -Dgpg.skip -Djacoco.skip -pl hudi-spark-datasource/hudi-spark
+```
+
+`HUDI_SDP_PYTHON` defaults to `python3`. Without usable Python dependencies the
+suite cancels its tests locally; `HUDI_SDP_REQUIRE_PYTHON=true` makes that a
failure
+and is always set in CI. Only the Spark 4.1 profile compiles the suite or
resolves
+the test-scoped Connect server dependency. Other Spark profiles are unaffected.
+The server binds an ephemeral loopback port and is stopped with the test
session.
+Each CLI has a 180-second timeout; dependency and post-failure probes have
shorter
+bounds. Generated pipelines and CLI logs remain under
+`hudi-spark-datasource/hudi-spark/target/sdp-layer2`, including after failures.
+
+## Run the layer-3 harness locally
+
+From the repository root, with `JAVA_HOME` pointing to JDK 17:
+
+```sh
+mvn package -T 2 -Pjava17 -Dspark4.1 -Dscala-2.13 \
+ -DskipTests -Dgpg.skip -Djacoco.skip -am -pl packaging/hudi-spark-bundle
+
+python3 -m venv /tmp/hudi-sdp-venv
+/tmp/hudi-sdp-venv/bin/python -m pip install \
+ -r hudi-integ-test/spark-pipelines/requirements.txt
+
+# Set this to the non-sources/non-javadoc jar produced by the build above.
+HUDI_BUNDLE="$PWD/packaging/hudi-spark-bundle/target/hudi-spark4.1-bundle_2.13-1.3.0-SNAPSHOT.jar"
+/tmp/hudi-sdp-venv/bin/python hudi-integ-test/spark-pipelines/harness.py \
+ --bundle "$HUDI_BUNDLE"
+```
+
+The last command starts the server, probes it with a remote query, runs the
+pipeline, reads the Hudi output, and stops the server. It uses the Spark server
+jars and executables installed by the pinned PySpark wheel; there is no
separate
+Spark distribution to download and no released Hudi bundle is substituted.
+An inherited `SPARK_HOME` or Spark defaults file is not used.
+
+`pipeline.py` declares one Hudi streaming table over a deterministic JSON input
+with three rows. SDP controls the AvailableNow execution and checkpoint
location.
+A second Connect client checks all business columns by both catalog name and
Hudi
+path, verifies `.hoodie/hoodie.properties`, and the harness checks that SDP
wrote
+a streaming offset under its checkpoint directory. Every run starts with new
+table, warehouse, input, Spark scratch, and checkpoint directories.
+
+This is infrastructure smoke coverage for
[#20402](https://github.com/apache/hudi/issues/20402).
+It does not establish the full RFC-112 contract: repeat materialized-view
refresh,
+streaming configuration forwarding, restart/full-refresh semantics, chaining,
+and keyed update/delete acceptance belong to the sibling tasks under
+[#20401](https://github.com/apache/hudi/issues/20401).
+
+## Configuration ownership
+
+| Configuration | Owner / location |
+| --- | --- |
+| Candidate bundle, Hudi session extension and `spark_catalog`, Kryo
serializer/registrator | `ConnectHarness.server_configuration()` and server
`spark-submit` arguments, before JVM startup |
+| Warehouse, in-memory catalog, local Hadoop filesystem, driver binding,
Connect port, scratch storage | Server startup configuration |
+| Pipeline storage/checkpoint root, catalog and database | Generated
`spark-pipeline.yml` |
+| Session shuffle partitions, SDP retry limit, input path | Pipeline YAML
`configuration` |
+| Flow-specific shuffle partitions | `@dp.table(spark_conf=...)` in
`pipeline.py` |
+| Hudi table type, record key, precombine field | Dataset `table_properties`
in `pipeline.py` |
+
+Static server settings must not be passed through the pipeline YAML or CLI
+`--conf`. The readiness client verifies the startup extension/catalog and that
+the extension is not session-modifiable. The CLI always receives
+`--remote sc://127.0.0.1:<port>`; it must not start its own local server.
+All data paths are absolute URIs accessible to the server and its local
executors.
+Tests for other storage backends must install connectors and configure storage
+access on the server at startup, before creating client sessions.
+
+## Failures and cleanup
+
+`--startup-timeout` (default 90 seconds) bounds the port wait and the readiness
+client individually. `--client-timeout` (default 180 seconds) bounds each CLI
or
+verification process, including Spark Connect retries. SDP flow retries are
+disabled in this smoke test to report the first failure. A server exit
interrupts
+a waiting client. All failures produce a nonzero harness exit code.
+
+Each subprocess starts in its own process group. On success, exceptions,
+timeouts, Ctrl-C, or SIGTERM, the harness sends SIGTERM and then SIGKILL after
a
+five-second grace period if necessary, reaps the launcher, and removes the test
+directories. It does not use `pkill`, Spark's shared PID files, or the global
+`stop-connect-server.sh`. A force-killed harness cannot run cleanup; the CI job
+has an outer timeout and runs on a disposable runner.
+
+Logs survive cleanup in a unique run directory under
`target/spark-pipelines-logs`
+(override with `--logs`). `server.log`, `probe.log`, `pipeline.log`,
`verify.log`,
+the server configuration, and the generated pipeline specification provide the
+execution context. On failure, the final 60 lines of each available process log
+are printed. CI uploads the log directory on both success and failure.
+
+## Extend the suite
+
+Use `ConnectHarness` as a context manager. Its `root` is isolated local
storage;
+`remote` is the explicit Connect URL. Write a scenario's definitions and spec
+under `root`, then call `run_pipeline(spec, name)` and `client(script, name)`.
+Each client script receives `name`, `remote`, and `root` as positional
arguments.
+Use distinct log names for successive calls. Spark calls belong in those
bounded
+client subprocesses, never in the supervising harness. Keep assertions in the
+scenario's verification client and let failures propagate.
+
+Fast lifecycle regression tests require only the Python standard library:
+
+```sh
+python3 -B -m unittest discover -s hudi-integ-test/spark-pipelines -p
'test_*.py' -v
+```
+
+The existing Scala-DML job in `.github/workflows/bot.yml` runs the lifecycle
tests,
+builds the candidate Spark 4.1 / Scala 2.13 bundle, and executes the
separate-server
+harness on every PR. The existing Scala-other job runs the layer-2 Maven suite.
+The commands above also support local validation before pushing.
diff --git a/hudi-integ-test/spark-pipelines/client.py
b/hudi-integ-test/spark-pipelines/client.py
new file mode 100644
index 000000000000..589961533de3
--- /dev/null
+++ b/hudi-integ-test/spark-pipelines/client.py
@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""All Spark operations here use a remote session; the harness bounds this
process."""
+
+from pathlib import Path
+import sys
+
+from pyspark.sql import SparkSession
+from pyspark.sql.connect.session import SparkSession as ConnectSession
+
+
+def main():
+ mode, remote, root_arg = sys.argv[1:]
+ root = Path(root_arg)
+ spark = SparkSession.builder.remote(remote).getOrCreate()
+ try:
+ assert isinstance(spark, ConnectSession), "Expected a remote Spark
Connect session"
+ assert spark.version == "4.1.1", spark.version
+ assert spark.conf.get("spark.hudi.test.run.id") == root.name,
"Connected to another test's server"
+ if mode == "probe":
+ assert spark.range(1).count() == 1
+ assert spark.conf.get("spark.sql.extensions") ==
"org.apache.spark.sql.hudi.HoodieSparkSessionExtension"
+ assert not spark.conf.isModifiable("spark.sql.extensions")
+ assert spark.conf.get("spark.sql.catalog.spark_catalog") ==
"org.apache.spark.sql.hudi.catalog.HoodieCatalog"
+ spark.sql("CREATE DATABASE sdp_test").collect()
+ print("Remote Spark 4.1.1 server ready with Hudi extension and
catalog")
+ elif mode == "verify":
+ expected = [(i, 100 + i, f"value-{i}") for i in range(1, 4)]
+ table = "spark_catalog.sdp_test.events"
+ actual = [tuple(row) for row in spark.table(table).select("id",
"ts", "value").orderBy("id").collect()]
+ assert actual == expected, (actual, expected)
+ location = root / "warehouse/sdp_test.db/events"
+ assert (location / ".hoodie/hoodie.properties").is_file(),
"Missing Hudi table metadata"
+ by_path =
spark.read.format("hudi").load(location.as_uri()).select("id", "ts",
"value").orderBy("id").collect()
+ assert [tuple(row) for row in by_path] == expected, by_path
+ print(f"Verified catalog and Hudi path reads: {actual}")
+ else:
+ raise ValueError(f"Unknown client mode: {mode}")
+ finally:
+ spark.stop()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/hudi-integ-test/spark-pipelines/harness.py
b/hudi-integ-test/spark-pipelines/harness.py
new file mode 100644
index 000000000000..a7978b590c1e
--- /dev/null
+++ b/hudi-integ-test/spark-pipelines/harness.py
@@ -0,0 +1,251 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Owned, foreground Spark Connect server and bounded SDP client processes
(POSIX)."""
+
+import argparse
+from collections import deque
+import contextlib
+import json
+import os
+from pathlib import Path
+import signal
+import socket
+import subprocess
+import sys
+import tempfile
+import time
+
+
+SPARK_VERSION = "4.1.1"
+HERE = Path(__file__).resolve().parent
+
+
[email protected]
+def owned_process(command, log, cwd, env):
+ """Kill only the new process group, including children left by an exited
launcher."""
+ with log.open("w") as output:
+ output.write("Command: " + repr([str(arg) for arg in command]) + "\n")
+ output.flush()
+ process = subprocess.Popen(command, cwd=cwd, env=env, stdout=output,
+ stderr=subprocess.STDOUT,
start_new_session=True)
+ try:
+ yield process
+ finally:
+ stop_process_group(process)
+
+
+def stop_process_group(process, grace=5):
+ # Retain the leader until its group has been signalled; never use pkill or
Spark's global stop script.
+ try:
+ os.killpg(process.pid, signal.SIGTERM)
+ except ProcessLookupError:
+ process.wait()
+ return
+ deadline = time.monotonic() + grace
+ while time.monotonic() < deadline:
+ process.poll()
+ try:
+ os.killpg(process.pid, 0)
+ except ProcessLookupError:
+ break
+ time.sleep(0.1)
+ else:
+ with contextlib.suppress(ProcessLookupError):
+ os.killpg(process.pid, signal.SIGKILL)
+ process.wait(timeout=grace)
+
+
+def wait_for_process(process, timeout, log, server=None):
+ deadline = time.monotonic() + timeout
+ while process.poll() is None:
+ if server is not None and server.poll() is not None:
+ raise RuntimeError(f"Connect server exited with
{server.returncode}; see server.log and {log}")
+ if time.monotonic() >= deadline:
+ raise TimeoutError(f"Client exceeded {timeout}s; see {log}")
+ time.sleep(0.1)
+ if process.returncode:
+ raise RuntimeError(f"Client exited with {process.returncode}; see
{log}")
+
+
+class ConnectHarness:
+ def __init__(self, bundle, logs, startup_timeout=90, client_timeout=180):
+ import pyspark
+
+ if pyspark.__version__ != SPARK_VERSION:
+ raise ValueError(f"Install requirements.txt: expected PySpark
{SPARK_VERSION}, got {pyspark.__version__}")
+ self.spark_home = Path(pyspark.__file__).resolve().parent
+ self.bundle = Path(bundle).resolve(strict=True)
+ if not self.bundle.is_file() or self.bundle.suffix != ".jar":
+ raise ValueError("--bundle must name the built Hudi Spark 4.1
bundle jar")
+ self.logs = Path(logs).resolve()
+ self.logs.mkdir(parents=True, exist_ok=True)
+ self.startup_timeout = startup_timeout
+ self.client_timeout = client_timeout
+ self.server = None
+ self.stack = contextlib.ExitStack()
+
+ def __enter__(self):
+ try:
+ self.root =
Path(self.stack.enter_context(tempfile.TemporaryDirectory(prefix="hudi-sdp-")))
+ with socket.socket() as sock:
+ sock.bind(("127.0.0.1", 0))
+ self.port = sock.getsockname()[1]
+ self.remote = f"sc://127.0.0.1:{self.port}"
+ self.env = self.process_environment()
+ conf = self.server_configuration()
+ (self.logs / "server-conf.json").write_text(json.dumps(conf,
indent=2) + "\n")
+ command = [str(self.spark_home / "bin/spark-submit"), "--master",
"local[2]",
+ "--driver-memory", "1g", "--jars", str(self.bundle)]
+ for key, value in conf.items():
+ command.extend(["--conf", f"{key}={value}"])
+ command.extend(["--class",
"org.apache.spark.sql.connect.service.SparkConnectServer"])
+ self.server = self.stack.enter_context(owned_process(
+ command, self.logs / "server.log", self.root, self.env))
+ # A TCP check alone can accept an unrelated listener. The probe
below also verifies our run ID.
+ self.wait_for_port()
+ self.client(HERE / "client.py", "probe",
timeout=self.startup_timeout)
+ return self
+ except BaseException:
+ self.stack.close()
+ raise
+
+ def __exit__(self, *exc):
+ self.stack.close()
+
+ def process_environment(self):
+ env = os.environ.copy()
+ for key in ("SPARK_REMOTE", "SPARK_CONNECT_MODE_ENABLED",
"SPARK_LOCAL_REMOTE", "SPARK_CONNECT_MODE",
+ "PYSPARK_SUBMIT_ARGS", "PYSPARK_GATEWAY_PORT",
"PYSPARK_GATEWAY_SECRET",
+ "SPARK_SUBMIT_OPTS", "SPARK_JAVA_OPTS", "PYTHONPATH"):
+ env.pop(key, None)
+ conf_dir = self.root / "conf"
+ conf_dir.mkdir()
+ env.update(SPARK_HOME=str(self.spark_home),
SPARK_CONF_DIR=str(conf_dir),
+ SPARK_LOCAL_IP="127.0.0.1", PYSPARK_PYTHON=sys.executable,
+ PYSPARK_DRIVER_PYTHON=sys.executable, PYTHONUNBUFFERED="1")
+ return env
+
+ def server_configuration(self):
+ # Static/driver/storage configuration belongs here, never in a
pipeline's session configuration.
+ return {
+ "spark.sql.extensions":
"org.apache.spark.sql.hudi.HoodieSparkSessionExtension",
+ "spark.sql.catalog.spark_catalog":
"org.apache.spark.sql.hudi.catalog.HoodieCatalog",
+ "spark.serializer": "org.apache.spark.serializer.KryoSerializer",
+ "spark.kryo.registrator":
"org.apache.spark.HoodieSparkKryoRegistrar",
+ "spark.sql.catalogImplementation": "in-memory",
+ "spark.sql.warehouse.dir": (self.root / "warehouse").as_uri(),
+ "spark.hadoop.fs.defaultFS": "file:///",
+ "spark.local.dir": str(self.root / "spark-local"),
+ "spark.driver.host": "127.0.0.1",
+ "spark.driver.bindAddress": "127.0.0.1",
+ "spark.connect.grpc.binding.address": "127.0.0.1",
+ "spark.connect.grpc.binding.port": str(self.port),
+ "spark.port.maxRetries": "0",
+ "spark.ui.enabled": "false",
+ "spark.hudi.test.run.id": self.root.name,
+ }
+
+ def wait_for_port(self):
+ deadline = time.monotonic() + self.startup_timeout
+ while time.monotonic() < deadline:
+ if self.server.poll() is not None:
+ raise RuntimeError(f"Connect server exited with
{self.server.returncode}; see server.log")
+ try:
+ with socket.create_connection(("127.0.0.1", self.port),
timeout=0.5):
+ return
+ except OSError:
+ time.sleep(0.2)
+ raise TimeoutError(f"Connect server startup exceeded
{self.startup_timeout}s; see server.log")
+
+ def run(self, command, name, timeout=None):
+ log = self.logs / f"{name}.log"
+ with owned_process(command, log, self.root, self.env) as process:
+ wait_for_process(process, timeout or self.client_timeout, log,
self.server)
+
+ def client(self, script, name, timeout=None):
+ self.run([sys.executable, str(script), name, self.remote,
str(self.root)], name, timeout)
+
+ def run_pipeline(self, spec, name="pipeline"):
+ self.run([str(self.spark_home / "bin/spark-pipelines"), "run",
"--remote", self.remote,
+ "--spec", str(spec)], name)
+
+ def smoke(self):
+ input_dir = self.root / "input"
+ input_dir.mkdir()
+ rows = [{"id": i, "ts": 100 + i, "value": f"value-{i}"} for i in
range(1, 4)]
+ (input_dir / "events.json").write_text("".join(json.dumps(row) + "\n"
for row in rows))
+ spec = {
+ "name": "hudi_connect_smoke",
+ "storage": (self.root / "checkpoints").as_uri(),
+ "catalog": "spark_catalog",
+ "database": "sdp_test",
+ "libraries": [{"glob": {"include": "pipeline.py"}}],
+ "configuration": {
+ "spark.sql.shuffle.partitions": "2",
+ "spark.sql.pipelines.maxFlowRetryAttempts": "0",
+ "hudi.test.input": input_dir.as_uri(),
+ },
+ }
+ (self.root / "pipeline.py").write_text((HERE /
"pipeline.py").read_text())
+ spec_path = self.root / "spark-pipeline.yml"
+ # JSON is a YAML subset; keep harness lifecycle tests independent of
third-party Python packages.
+ spec_path.write_text(json.dumps(spec, indent=2) + "\n")
+ (self.logs / spec_path.name).write_text(spec_path.read_text())
+ self.run_pipeline(spec_path)
+ self.client(HERE / "client.py", "verify")
+ if not list((self.root / "checkpoints").rglob("offsets/0")):
+ raise AssertionError("SDP did not create a streaming checkpoint
under its isolated storage directory")
+
+
+def positive_seconds(value):
+ seconds = int(value)
+ if seconds <= 0:
+ raise argparse.ArgumentTypeError("timeout must be positive")
+ return seconds
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--bundle", required=True, type=Path)
+ parser.add_argument("--logs", type=Path,
default=Path("target/spark-pipelines-logs"))
+ parser.add_argument("--startup-timeout", type=positive_seconds, default=90)
+ parser.add_argument("--client-timeout", type=positive_seconds, default=180)
+ args = parser.parse_args()
+ args.logs.mkdir(parents=True, exist_ok=True)
+ logs = Path(tempfile.mkdtemp(prefix="run-", dir=args.logs.resolve()))
+ print(f"Spark Connect SDP logs: {logs}", flush=True)
+ # Let CI cancellation reach ExitStack, just like Ctrl-C; each child has
its own process group.
+ def interrupted(signum, frame):
+ raise KeyboardInterrupt(f"Received signal {signum}")
+ signal.signal(signal.SIGTERM, interrupted)
+ try:
+ with ConnectHarness(args.bundle, logs, args.startup_timeout,
args.client_timeout) as harness:
+ harness.smoke()
+ except (Exception, KeyboardInterrupt) as error:
+ print(f"FAILED: {error}", file=sys.stderr)
+ for log in sorted(logs.glob("*.log")):
+ print(f"\n--- {log.name} (last 60 lines) ---", file=sys.stderr)
+ with log.open(errors="replace") as output:
+ print("".join(deque(output, maxlen=60)), file=sys.stderr)
+ return 1
+ print("PASS: spark-pipelines wrote the expected Hudi rows through Spark
Connect; resources removed.")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/hudi-integ-test/spark-pipelines/pipeline.py
b/hudi-integ-test/spark-pipelines/pipeline.py
new file mode 100644
index 000000000000..57926915b3bf
--- /dev/null
+++ b/hudi-integ-test/spark-pipelines/pipeline.py
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from pyspark import pipelines as dp
+from pyspark.sql import SparkSession
+
+spark = SparkSession.active()
+
+
[email protected](name="events", format="hudi", table_properties={
+ "type": "cow", "primaryKey": "id", "preCombineField": "ts",
+}, spark_conf={"spark.sql.shuffle.partitions": "2"})
+def events():
+ return spark.readStream.schema("id INT, ts LONG, value
STRING").json(spark.conf.get("hudi.test.input"))
diff --git a/hudi-integ-test/spark-pipelines/requirements.txt
b/hudi-integ-test/spark-pipelines/requirements.txt
new file mode 100644
index 000000000000..f27c90ed6b76
--- /dev/null
+++ b/hudi-integ-test/spark-pipelines/requirements.txt
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+pyspark[pipelines,connect]==4.1.1
diff --git a/hudi-integ-test/spark-pipelines/test_harness.py
b/hudi-integ-test/spark-pipelines/test_harness.py
new file mode 100644
index 000000000000..a2969594f51b
--- /dev/null
+++ b/hudi-integ-test/spark-pipelines/test_harness.py
@@ -0,0 +1,98 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Fast lifecycle regressions; no Spark installation is needed for these
tests."""
+
+import os
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import time
+from types import SimpleNamespace
+import unittest
+from unittest.mock import patch
+
+from harness import ConnectHarness, owned_process, stop_process_group,
wait_for_process
+
+
+class TestProcessLifecycle(unittest.TestCase):
+ def setUp(self):
+ self.temp = tempfile.TemporaryDirectory()
+ self.addCleanup(self.temp.cleanup)
+ self.root = Path(self.temp.name)
+ self.log = self.root / "client.log"
+
+ def process(self, code):
+ return owned_process([sys.executable, "-u", "-c", code], self.log,
self.root, os.environ.copy())
+
+ def test_client_failure_keeps_diagnostics(self):
+ with self.process("print('intentional client failure'); raise
SystemExit(7)") as process:
+ with self.assertRaisesRegex(RuntimeError, "exited with 7"):
+ wait_for_process(process, 5, self.log)
+ self.assertIn("intentional client failure", self.log.read_text())
+
+ def
test_timeout_terminates_owned_process_without_stopping_another_server(self):
+ other = subprocess.Popen([sys.executable, "-c", "import time;
time.sleep(60)"], start_new_session=True)
+ self.addCleanup(stop_process_group, other)
+ with self.assertRaises(TimeoutError):
+ with self.process("import time; time.sleep(60)") as process:
+ wait_for_process(process, 0.2, self.log)
+ self.assertIsNotNone(process.poll())
+ self.assertIsNone(other.poll())
+
+ def test_server_exit_interrupts_client(self):
+ server = subprocess.Popen([sys.executable, "-c", "raise
SystemExit(9)"])
+ server.wait(timeout=5)
+ started = time.monotonic()
+ with self.assertRaisesRegex(RuntimeError, "server exited with 9"):
+ with self.process("import time; time.sleep(60)") as process:
+ wait_for_process(process, 30, self.log, server)
+ self.assertLess(time.monotonic() - started, 10)
+ self.assertIsNotNone(process.poll())
+
+ def test_kills_process_ignoring_term(self):
+ command = [sys.executable, "-u", "-c",
+ "import signal,time; signal.signal(signal.SIGTERM,
signal.SIG_IGN); print('ready'); time.sleep(60)"]
+ process = subprocess.Popen(command, stdout=subprocess.PIPE, text=True,
start_new_session=True)
+ self.addCleanup(stop_process_group, process, 0.2)
+ self.addCleanup(process.stdout.close)
+ self.assertEqual(process.stdout.readline().strip(), "ready")
+ stop_process_group(process, grace=0.2)
+ self.assertEqual(process.returncode, -9)
+
+ def test_startup_failure_cleans_server_and_test_resources(self):
+ spark_home = self.root / "pyspark"
+ (spark_home / "bin").mkdir(parents=True)
+ submit = spark_home / "bin/spark-submit"
+ submit.write_text(f"#!{sys.executable}\nimport time\ntime.sleep(60)\n")
+ submit.chmod(0o755)
+ bundle = self.root / "bundle.jar"
+ bundle.touch()
+ fake_pyspark = SimpleNamespace(__version__="4.1.1",
__file__=str(spark_home / "__init__.py"))
+ with patch.dict(sys.modules, pyspark=fake_pyspark):
+ harness = ConnectHarness(bundle, self.root / "logs",
startup_timeout=0.2)
+ with self.assertRaisesRegex(TimeoutError, "startup exceeded"):
+ with harness:
+ self.fail("A server which never listens must not become ready")
+ self.assertFalse(harness.root.exists())
+ self.assertIsNotNone(harness.server.poll())
+ self.assertTrue((harness.logs / "server.log").is_file())
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/hudi-spark-datasource/hudi-spark/pom.xml
b/hudi-spark-datasource/hudi-spark/pom.xml
index 8b3daf01fdfe..ca4a1fd4a29b 100644
--- a/hudi-spark-datasource/hudi-spark/pom.xml
+++ b/hudi-spark-datasource/hudi-spark/pom.xml
@@ -525,6 +525,54 @@
</dependencies>
<profiles>
+ <!-- SDP is a Spark 4.1 API. Keep its server dependency and tests out of
other profiles. -->
+ <profile>
+ <id>spark4.1</id>
+ <activation>
+ <property><name>spark4.1</name></property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-connect_${scala.binary.version}</artifactId>
+ <version>${spark41.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>add-sdp-test-source</id>
+ <phase>generate-test-sources</phase>
+ <goals><goal>add-test-source</goal></goals>
+ <configuration>
+ <sources><source>src/test/scala-spark4.1</source></sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.scalastyle</groupId>
+ <artifactId>scalastyle-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>check-sdp-test-style</id>
+ <phase>test-compile</phase>
+ <goals><goal>check</goal></goals>
+ <configuration>
+
<testSourceDirectory>${project.basedir}/src/test/scala-spark4.1</testSourceDirectory>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+
<!-- Vortex requires Java 17 (vortex-jni/vortex-spark ship Java 17
bytecode). The Vortex
round-trip test compiles/runs only under JDK 17; Java 11 builds skip
it. -->
<profile>
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala-spark4.1/org/apache/spark/sql/connect/hudi/TestHoodieSparkConnectPipelines.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala-spark4.1/org/apache/spark/sql/connect/hudi/TestHoodieSparkConnectPipelines.scala
new file mode 100644
index 000000000000..5436d3efac97
--- /dev/null
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala-spark4.1/org/apache/spark/sql/connect/hudi/TestHoodieSparkConnectPipelines.scala
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connect.hudi
+
+import org.apache.spark.sql.{Row, SparkSession}
+import org.apache.spark.sql.connect.service.SparkConnectService
+import org.apache.spark.util.Utils
+import org.scalatest.{BeforeAndAfterAll, FunSuite}
+
+import java.io.{File, IOException}
+import java.nio.charset.StandardCharsets.UTF_8
+import java.nio.file.Files
+import java.util.UUID
+import java.util.concurrent.TimeUnit
+
+/** Real pipelines CLI against a Connect service in the test JVM, without
building a bundle. */
+class TestHoodieSparkConnectPipelines extends FunSuite with BeforeAndAfterAll {
+ private val python = sys.env.getOrElse("HUDI_SDP_PYTHON", "python3")
+ private val logs = new File("target/sdp-layer2", UUID.randomUUID().toString)
+ private var spark: SparkSession = _
+ private var root: File = _
+ private var started = false
+
+ private lazy val pythonAvailable: Boolean = {
+ logs.mkdirs()
+ val probe = "import pyspark; from pyspark.pipelines import cli; " +
+ "from pyspark.sql.connect.session import SparkSession; " +
+ "assert pyspark.__version__ == '4.1.1', pyspark.__version__"
+ try {
+ runPython(Seq("-c", probe), "python-probe", 30)._1 == 0
+ } catch {
+ case _: IOException => false
+ }
+ }
+
+ private def startServer(): Unit = {
+ if (sys.env.get("HUDI_SDP_REQUIRE_PYTHON").contains("true")) {
+ assert(pythonAvailable, s"CI requires PySpark 4.1.1 pipelines/connect;
see $logs")
+ } else {
+ assume(pythonAvailable, "Install the SDP requirements and set
HUDI_SDP_PYTHON to enable this suite")
+ }
+ if (!started) {
+ root = Utils.createTempDir(namePrefix = "hudi-sdp-layer2")
+ spark = SparkSession.builder().master("local[2]").appName("Hudi SDP CLI
tests")
+ .config("spark.ui.enabled", "false")
+ .config("spark.sql.warehouse.dir", new File(root,
"warehouse").toURI.toString)
+ .config("spark.sql.catalogImplementation", "in-memory")
+ .config("spark.sql.extensions",
"org.apache.spark.sql.hudi.HoodieSparkSessionExtension")
+ .config("spark.sql.catalog.spark_catalog",
"org.apache.spark.sql.hudi.catalog.HoodieCatalog")
+ .config("spark.serializer",
"org.apache.spark.serializer.KryoSerializer")
+ .config("spark.kryo.registrator",
"org.apache.spark.HoodieSparkKryoRegistrar")
+ .config("spark.connect.grpc.binding.address", "127.0.0.1")
+ .config("spark.connect.grpc.binding.port", "0")
+ .getOrCreate()
+ SparkConnectService.start(spark.sparkContext)
+ started = true
+ }
+ }
+
+ test("CLI materialized view and streaming table write exact Hudi rows") {
+ withPipeline { (dir, database) =>
+ write(dir, "flows.sql",
+ """CREATE MATERIALIZED VIEW mv USING hudi
+ |TBLPROPERTIES ('type'='cow', 'primaryKey'='id',
'preCombineField'='ts')
+ |AS SELECT 2 AS id, 2L AS ts, 'mv' AS value;
+ |CREATE STREAMING TABLE events USING hudi
+ |TBLPROPERTIES ('type'='cow', 'primaryKey'='id',
'preCombineField'='ts')
+ |AS SELECT * FROM STREAM input_events;
+ |""".stripMargin)
+ assertSuccess(runPipeline(dir, database))
+ assert(rows(database, "mv") == Seq(Row(2, 2L, "mv")))
+ assert(rows(database, "events") == Seq(Row(1, 1L, "old")))
+ }
+ }
+
+ test("flow-scoped SQL settings independently select insert and upsert") {
+ withPipeline { (dir, database) =>
+ write(dir, "flows.sql", Seq("insert", "upsert").map { operation =>
+ s"""SET spark.hoodie.datasource.write.operation=$operation;
+ |CREATE STREAMING TABLE ${operation}_events USING hudi
+ |TBLPROPERTIES ('type'='cow', 'primaryKey'='id',
'preCombineField'='ts')
+ |AS SELECT * FROM STREAM input_events;
+ |""".stripMargin
+ }.mkString("\n"))
+ assertSuccess(runPipeline(dir, database))
+ assert(rows(database, "insert_events") == Seq.fill(2)(Row(1, 1L, "old")))
+ assert(rows(database, "upsert_events") == Seq(Row(1, 1L, "old")))
+ }
+ }
+
+ test("a failing CLI flow reports the key error and leaves Connect usable") {
+ withPipeline { (dir, database) =>
+ write(dir, "flows.sql",
+ """CREATE STREAMING TABLE bad_events USING hudi
+ |TBLPROPERTIES ('type'='cow', 'primaryKey'='id',
'preCombineField'='ts')
+ |AS SELECT CAST(NULL AS INT) AS id, ts, value FROM STREAM
input_events;
+ |""".stripMargin)
+ val (exit, output) = runPipeline(dir, database)
+ assert(exit != 0, "The null-key flow unexpectedly succeeded")
+ assert(output.contains("recordKey") && output.contains("null"),
output.takeRight(12000))
+ val probe = "from pyspark.sql import SparkSession; " +
+ s"s = SparkSession.builder.remote('$remote').getOrCreate(); " +
+ "assert s.range(3).count() == 3; s.stop()"
+ assertSuccess(runPython(Seq("-c", probe), s"$database-after-failure",
60))
+ }
+ }
+
+ private def remote: String =
s"sc://127.0.0.1:${SparkConnectService.localPort}"
+
+ private def withPipeline(f: (File, String) => Unit): Unit = {
+ startServer()
+ val database = "sdp_" + UUID.randomUUID().toString.replace("-", "")
+ val dir = new File(root, database)
+ dir.mkdirs()
+ spark.sql(s"CREATE DATABASE $database").collect()
+ val input = new File(dir, "input")
+ input.mkdirs()
+ write(input, "events.json", """{"id":1,"ts":1,"value":"old"}
+ |{"id":1,"ts":1,"value":"old"}
+ |""".stripMargin)
+ write(dir, "source.py",
+ """from pyspark import pipelines as dp
+ |from pyspark.sql import SparkSession
+ |spark = SparkSession.active()
+ |@dp.temporary_view(name="input_events")
+ |def input_events():
+ | return spark.readStream.schema("id INT, ts LONG, value
STRING").json(spark.conf.get("hudi.test.input"))
+ |""".stripMargin)
+ try f(dir, database) finally {
+ spark.sql(s"DROP DATABASE $database CASCADE").collect()
+ }
+ }
+
+ private def runPipeline(dir: File, database: String): (Int, String) = {
+ def quoted(value: String): String = "'" + value.replace("'", "''") + "'"
+ write(dir, "spark-pipeline.yml",
+ s"""name: $database
+ |catalog: spark_catalog
+ |database: $database
+ |storage: ${quoted(new File(dir, "checkpoints").toURI.toString)}
+ |libraries:
+ | - glob: {include: source.py}
+ | - glob: {include: flows.sql}
+ |configuration:
+ | hudi.test.input: ${quoted(new File(dir, "input").toURI.toString)}
+ | spark.sql.shuffle.partitions: '2'
+ | spark.sql.pipelines.maxFlowRetryAttempts: '0'
+ | spark.hoodie.metadata.enable: 'true'
+ |""".stripMargin)
+ Seq("source.py", "flows.sql", "spark-pipeline.yml").foreach { name =>
+ Files.copy(new File(dir, name).toPath, new File(logs,
s"$database-$name").toPath)
+ }
+ runPython(Seq("-m", "pyspark.pipelines.cli", "run",
+ "--spec", new File(dir, "spark-pipeline.yml").getAbsolutePath),
database, 180)
+ }
+
+ private def rows(database: String, table: String): Seq[Row] = {
+ spark.table(s"$database.$table").select("id", "ts", "value").orderBy("id",
"ts").collect().toSeq
+ }
+
+ private def write(dir: File, name: String, value: String): Unit = {
+ Files.write(new File(dir, name).toPath, value.getBytes(UTF_8))
+ }
+
+ private def assertSuccess(result: (Int, String)): Unit = {
+ assert(result._1 == 0, result._2.takeRight(12000))
+ }
+
+ private def runPython(args: Seq[String], name: String, timeout: Long): (Int,
String) = {
+ val log = new File(logs, s"$name.log")
+ val builder = new ProcessBuilder((Seq(python) ++ args): _*)
+ .redirectErrorStream(true).redirectOutput(log)
+ Seq("SPARK_REMOTE", "SPARK_CONNECT_MODE_ENABLED", "SPARK_LOCAL_REMOTE",
"SPARK_CONNECT_MODE",
+ "PYSPARK_GATEWAY_PORT", "PYSPARK_GATEWAY_SECRET",
"PYTHONPATH").foreach(builder.environment().remove)
+ builder.environment().put("PYTHONUNBUFFERED", "1")
+ // The spark-pipelines launcher translates --remote into SPARK_REMOTE
before invoking cli.py.
+ if (started) builder.environment().put("SPARK_REMOTE", remote)
+ val process = builder.start()
+ try {
+ assert(process.waitFor(timeout, TimeUnit.SECONDS), s"CLI timed out after
${timeout}s; see $log")
+ (process.exitValue(), new String(Files.readAllBytes(log.toPath), UTF_8))
+ } finally {
+ process.destroy()
+ if (!process.waitFor(5, TimeUnit.SECONDS)) {
+ process.destroyForcibly()
+ process.waitFor(5, TimeUnit.SECONDS)
+ }
+ }
+ }
+
+ override protected def afterAll(): Unit = {
+ try {
+ if (spark != null) {
+ try SparkConnectService.stop() finally spark.stop()
+ }
+ } finally {
+ if (root != null) Utils.deleteRecursively(root)
+ SparkSession.clearActiveSession()
+ SparkSession.clearDefaultSession()
+ super.afterAll()
+ }
+ }
+}