This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/master/pr-20453-7522e3af8dc46712e69d1ef1327df31decd5ace7 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 6d3b743b10151272569ea9f01558a41794885df4 Author: Danny Chan <[email protected]> AuthorDate: Mon Nov 16 06:00:01 2026 +0000 test(spark): add Spark Connect SDP integration harness and CI coverage (#20453) * test(spark): add Spark Connect SDP integration harness and CI coverage * test(spark): split SDP CLI coverage between Maven and nightly * test(spark): run SDP checks in existing CI jobs --- .github/workflows/bot.yml | 70 ++++++ hudi-integ-test/spark-pipelines/README.md | 172 ++++++++++++++ hudi-integ-test/spark-pipelines/client.py | 59 +++++ hudi-integ-test/spark-pipelines/harness.py | 251 +++++++++++++++++++++ hudi-integ-test/spark-pipelines/pipeline.py | 28 +++ hudi-integ-test/spark-pipelines/requirements.txt | 18 ++ hudi-integ-test/spark-pipelines/test_harness.py | 98 ++++++++ hudi-spark-datasource/hudi-spark/pom.xml | 48 ++++ .../hudi/TestHoodieSparkConnectPipelines.scala | 218 ++++++++++++++++++ 9 files changed, 962 insertions(+) diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml index 923d0056e812..dcdf98a7cdfa 100644 --- a/.github/workflows/bot.yml +++ b/.github/workflows/bot.yml @@ -955,6 +955,42 @@ jobs: with: flags: spark-scala-tests token: ${{ secrets.CODECOV_TOKEN }} + # Run SDP only in the existing Spark 4.2 matrix entry, after its tests and + # coverage upload. A clean Spark 4.1 rebuild prevents mixed-profile classes. + - name: Set up Python for SDP + if: matrix.sparkProfile == 'spark4.2' + uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: pip + cache-dependency-path: hudi-integ-test/spark-pipelines/requirements.txt + - name: Install pinned SDP CLI dependencies + if: matrix.sparkProfile == 'spark4.2' + run: python3 -m pip install -r hudi-integ-test/spark-pipelines/requirements.txt + - name: Build Spark 4.1 bundle and run SDP process tests + if: matrix.sparkProfile == 'spark4.2' + timeout-minutes: 25 + run: | + python3 -B -m unittest discover -s hudi-integ-test/spark-pipelines -p 'test_harness.py' -v + mvn clean package -T 2 -Pjava17 -Dspark4.1 -Dscala-2.13 -DskipTests \ + $MVN_ARGS -am -pl packaging/hudi-spark-bundle + shopt -s nullglob + bundles=(packaging/hudi-spark-bundle/target/hudi-spark4.1-bundle_2.13-*.jar) + candidates=() + for bundle in "${bundles[@]}"; do + [[ "$bundle" == *-sources.jar || "$bundle" == *-javadoc.jar ]] || candidates+=("$bundle") + done + [[ ${#candidates[@]} -eq 1 ]] || { echo "Expected one candidate bundle, found ${#candidates[@]}"; exit 1; } + python3 hudi-integ-test/spark-pipelines/harness.py \ + --bundle "${candidates[0]}" --logs "$RUNNER_TEMP/sdp-logs" + - name: Upload SDP process diagnostics + if: always() && matrix.sparkProfile == 'spark4.2' + uses: actions/upload-artifact@v4 + with: + name: spark-connect-layer3-logs + path: ${{ runner.temp }}/sdp-logs + if-no-files-found: ignore + test-spark-java17-scala-other-tests: if: *route @@ -1016,6 +1052,40 @@ jobs: with: flags: spark-scala-tests token: ${{ secrets.CODECOV_TOKEN }} + # Run SDP only in the existing Spark 4.2 matrix entry, after its tests and + # coverage upload. A clean Spark 4.1 rebuild prevents mixed-profile classes. + - name: Set up Python for SDP + if: matrix.sparkProfile == 'spark4.2' + uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: pip + cache-dependency-path: hudi-integ-test/spark-pipelines/requirements.txt + - name: Install pinned SDP CLI dependencies + if: matrix.sparkProfile == 'spark4.2' + run: python3 -m pip install -r hudi-integ-test/spark-pipelines/requirements.txt + - name: Build Spark 4.1 dependencies and run SDP ScalaTests + if: matrix.sparkProfile == 'spark4.2' + timeout-minutes: 25 + env: + HUDI_SDP_REQUIRE_PYTHON: 'true' + run: | + mvn clean install -T 2 -Pjava17 -Dspark4.1 -Dscala-2.13 -DskipTests \ + $MVN_ARGS -am -pl hudi-spark-datasource/hudi-spark + mvn test -Punit-tests -Pjava17 -Dspark4.1 -Dscala-2.13 \ + -Dtest=skipJavaTests -Dsurefire.failIfNoSpecifiedTests=false \ + -Dsuites=org.apache.spark.sql.connect.hudi.TestHoodieSparkConnectPipelines \ + $MVN_ARGS -pl hudi-spark-datasource/hudi-spark + - name: Upload SDP ScalaTest diagnostics + if: always() && matrix.sparkProfile == 'spark4.2' + uses: actions/upload-artifact@v4 + with: + name: spark-connect-layer2-logs + path: | + hudi-spark-datasource/hudi-spark/target/sdp-layer2 + hudi-spark-datasource/hudi-spark/target/surefire-reports + if-no-files-found: ignore + # [CI-TRIM] to revisit for CI improvement # build-spark-java17: diff --git a/hudi-integ-test/spark-pipelines/README.md b/hudi-integ-test/spark-pipelines/README.md new file mode 100644 index 000000000000..00c65567cba1 --- /dev/null +++ b/hudi-integ-test/spark-pipelines/README.md @@ -0,0 +1,172 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +# Spark Connect / Declarative Pipelines integration harness + +The process harness runs the **real `spark-pipelines` CLI** against a separately launched +Spark 4.1.1 Connect JVM with the candidate Hudi bundle. It supports Linux and +macOS, JDK 17, and Python 3.10 or newer (CI uses Python 3.11). +The server and clients share a local filesystem; no external storage or metastore +is required. Use a dedicated Python virtual environment. + +## Two test layers + +Layer 2 invokes `python -m pyspark.pipelines.cli` with `SPARK_REMOTE`; layer 3 +also exercises the `spark-pipelines` launcher. Both use PySpark 4.1.1. Neither substitutes +in-JVM `DataflowGraph` calls for the client/server protocol. + +| Layer | Server | Coverage | Execution | +| --- | --- | --- | --- | +| 2: `TestHoodieSparkConnectPipelines` | Connect service in the ScalaTest JVM, Hudi on the Maven test classpath | MV/ST exact rows, independent SQL flow insert/upsert settings, null-key failure and a fresh remote query afterward | Existing Scala-other job on PRs, pushes and merge groups | +| 3: `harness.py` | Separate `spark-submit` JVM with the candidate bundle | Bundle packaging, static server configuration, remote writes/reads, process ownership and cleanup | Existing Scala-DML job on PRs, pushes and merge groups | + +Both layers run as additional steps in existing `bot.yml` jobs; no new workflow, +job, or matrix entry is added. Layer 2 runs in `test-spark-java17-scala-other-tests` +and layer 3 in `test-spark-java17-scala-dml-tests`, only in their existing Spark 4.2 +matrix entries. Each first completes its normal tests and coverage upload, then +performs a clean Spark 4.1 rebuild so compiled classes cannot leak across profiles. +Both jobs already feed `Java CI result`; an SDP failure therefore fails normal PR +validation. The separate nightly workflow and its retry policy are removed. + +The November 11 guidance from Ethan and Siva permits every-PR execution with less +than 30 minutes of additional runtime per job. Python setup/install, the Spark +4.1 build and the tests must be included in that measurement. The build/test step +has a 25-minute timeout to leave headroom; a timeout is a failure, not evidence +that the performance target was met. CI timings must confirm the complete added +cost independently for each job. There is no combined runtime budget across the +two jobs and no new global nightly gate or bypass mechanism. + +### Run the layer-2 suite locally + +With JDK 17, install the pinned `requirements.txt` in a Python environment, then: + +```sh +mvn install -T 2 -Pjava17 -Dspark4.1 -Dscala-2.13 -DskipTests \ + -Dgpg.skip -Djacoco.skip -am -pl hudi-spark-datasource/hudi-spark +HUDI_SDP_PYTHON=/path/to/venv/bin/python HUDI_SDP_REQUIRE_PYTHON=true \ + mvn test -Punit-tests -Pjava17 -Dspark4.1 -Dscala-2.13 \ + -Dtest=skipJavaTests -Dsurefire.failIfNoSpecifiedTests=false \ + -Dsuites=org.apache.spark.sql.connect.hudi.TestHoodieSparkConnectPipelines \ + -Dgpg.skip -Djacoco.skip -pl hudi-spark-datasource/hudi-spark +``` + +`HUDI_SDP_PYTHON` defaults to `python3`. Without usable Python dependencies the +suite cancels its tests locally; `HUDI_SDP_REQUIRE_PYTHON=true` makes that a failure +and is always set in CI. Only the Spark 4.1 profile compiles the suite or resolves +the test-scoped Connect server dependency. Other Spark profiles are unaffected. +The server binds an ephemeral loopback port and is stopped with the test session. +Each CLI has a 180-second timeout; dependency and post-failure probes have shorter +bounds. Generated pipelines and CLI logs remain under +`hudi-spark-datasource/hudi-spark/target/sdp-layer2`, including after failures. + +## Run the layer-3 harness locally + +From the repository root, with `JAVA_HOME` pointing to JDK 17: + +```sh +mvn package -T 2 -Pjava17 -Dspark4.1 -Dscala-2.13 \ + -DskipTests -Dgpg.skip -Djacoco.skip -am -pl packaging/hudi-spark-bundle + +python3 -m venv /tmp/hudi-sdp-venv +/tmp/hudi-sdp-venv/bin/python -m pip install \ + -r hudi-integ-test/spark-pipelines/requirements.txt + +# Set this to the non-sources/non-javadoc jar produced by the build above. +HUDI_BUNDLE="$PWD/packaging/hudi-spark-bundle/target/hudi-spark4.1-bundle_2.13-1.3.0-SNAPSHOT.jar" +/tmp/hudi-sdp-venv/bin/python hudi-integ-test/spark-pipelines/harness.py \ + --bundle "$HUDI_BUNDLE" +``` + +The last command starts the server, probes it with a remote query, runs the +pipeline, reads the Hudi output, and stops the server. It uses the Spark server +jars and executables installed by the pinned PySpark wheel; there is no separate +Spark distribution to download and no released Hudi bundle is substituted. +An inherited `SPARK_HOME` or Spark defaults file is not used. + +`pipeline.py` declares one Hudi streaming table over a deterministic JSON input +with three rows. SDP controls the AvailableNow execution and checkpoint location. +A second Connect client checks all business columns by both catalog name and Hudi +path, verifies `.hoodie/hoodie.properties`, and the harness checks that SDP wrote +a streaming offset under its checkpoint directory. Every run starts with new +table, warehouse, input, Spark scratch, and checkpoint directories. + +This is infrastructure smoke coverage for [#20402](https://github.com/apache/hudi/issues/20402). +It does not establish the full RFC-112 contract: repeat materialized-view refresh, +streaming configuration forwarding, restart/full-refresh semantics, chaining, +and keyed update/delete acceptance belong to the sibling tasks under +[#20401](https://github.com/apache/hudi/issues/20401). + +## Configuration ownership + +| Configuration | Owner / location | +| --- | --- | +| Candidate bundle, Hudi session extension and `spark_catalog`, Kryo serializer/registrator | `ConnectHarness.server_configuration()` and server `spark-submit` arguments, before JVM startup | +| Warehouse, in-memory catalog, local Hadoop filesystem, driver binding, Connect port, scratch storage | Server startup configuration | +| Pipeline storage/checkpoint root, catalog and database | Generated `spark-pipeline.yml` | +| Session shuffle partitions, SDP retry limit, input path | Pipeline YAML `configuration` | +| Flow-specific shuffle partitions | `@dp.table(spark_conf=...)` in `pipeline.py` | +| Hudi table type, record key, precombine field | Dataset `table_properties` in `pipeline.py` | + +Static server settings must not be passed through the pipeline YAML or CLI +`--conf`. The readiness client verifies the startup extension/catalog and that +the extension is not session-modifiable. The CLI always receives +`--remote sc://127.0.0.1:<port>`; it must not start its own local server. +All data paths are absolute URIs accessible to the server and its local executors. +Tests for other storage backends must install connectors and configure storage +access on the server at startup, before creating client sessions. + +## Failures and cleanup + +`--startup-timeout` (default 90 seconds) bounds the port wait and the readiness +client individually. `--client-timeout` (default 180 seconds) bounds each CLI or +verification process, including Spark Connect retries. SDP flow retries are +disabled in this smoke test to report the first failure. A server exit interrupts +a waiting client. All failures produce a nonzero harness exit code. + +Each subprocess starts in its own process group. On success, exceptions, +timeouts, Ctrl-C, or SIGTERM, the harness sends SIGTERM and then SIGKILL after a +five-second grace period if necessary, reaps the launcher, and removes the test +directories. It does not use `pkill`, Spark's shared PID files, or the global +`stop-connect-server.sh`. A force-killed harness cannot run cleanup; the CI job +has an outer timeout and runs on a disposable runner. + +Logs survive cleanup in a unique run directory under `target/spark-pipelines-logs` +(override with `--logs`). `server.log`, `probe.log`, `pipeline.log`, `verify.log`, +the server configuration, and the generated pipeline specification provide the +execution context. On failure, the final 60 lines of each available process log +are printed. CI uploads the log directory on both success and failure. + +## Extend the suite + +Use `ConnectHarness` as a context manager. Its `root` is isolated local storage; +`remote` is the explicit Connect URL. Write a scenario's definitions and spec +under `root`, then call `run_pipeline(spec, name)` and `client(script, name)`. +Each client script receives `name`, `remote`, and `root` as positional arguments. +Use distinct log names for successive calls. Spark calls belong in those bounded +client subprocesses, never in the supervising harness. Keep assertions in the +scenario's verification client and let failures propagate. + +Fast lifecycle regression tests require only the Python standard library: + +```sh +python3 -B -m unittest discover -s hudi-integ-test/spark-pipelines -p 'test_*.py' -v +``` + +The existing Scala-DML job in `.github/workflows/bot.yml` runs the lifecycle tests, +builds the candidate Spark 4.1 / Scala 2.13 bundle, and executes the separate-server +harness on every PR. The existing Scala-other job runs the layer-2 Maven suite. +The commands above also support local validation before pushing. diff --git a/hudi-integ-test/spark-pipelines/client.py b/hudi-integ-test/spark-pipelines/client.py new file mode 100644 index 000000000000..589961533de3 --- /dev/null +++ b/hudi-integ-test/spark-pipelines/client.py @@ -0,0 +1,59 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""All Spark operations here use a remote session; the harness bounds this process.""" + +from pathlib import Path +import sys + +from pyspark.sql import SparkSession +from pyspark.sql.connect.session import SparkSession as ConnectSession + + +def main(): + mode, remote, root_arg = sys.argv[1:] + root = Path(root_arg) + spark = SparkSession.builder.remote(remote).getOrCreate() + try: + assert isinstance(spark, ConnectSession), "Expected a remote Spark Connect session" + assert spark.version == "4.1.1", spark.version + assert spark.conf.get("spark.hudi.test.run.id") == root.name, "Connected to another test's server" + if mode == "probe": + assert spark.range(1).count() == 1 + assert spark.conf.get("spark.sql.extensions") == "org.apache.spark.sql.hudi.HoodieSparkSessionExtension" + assert not spark.conf.isModifiable("spark.sql.extensions") + assert spark.conf.get("spark.sql.catalog.spark_catalog") == "org.apache.spark.sql.hudi.catalog.HoodieCatalog" + spark.sql("CREATE DATABASE sdp_test").collect() + print("Remote Spark 4.1.1 server ready with Hudi extension and catalog") + elif mode == "verify": + expected = [(i, 100 + i, f"value-{i}") for i in range(1, 4)] + table = "spark_catalog.sdp_test.events" + actual = [tuple(row) for row in spark.table(table).select("id", "ts", "value").orderBy("id").collect()] + assert actual == expected, (actual, expected) + location = root / "warehouse/sdp_test.db/events" + assert (location / ".hoodie/hoodie.properties").is_file(), "Missing Hudi table metadata" + by_path = spark.read.format("hudi").load(location.as_uri()).select("id", "ts", "value").orderBy("id").collect() + assert [tuple(row) for row in by_path] == expected, by_path + print(f"Verified catalog and Hudi path reads: {actual}") + else: + raise ValueError(f"Unknown client mode: {mode}") + finally: + spark.stop() + + +if __name__ == "__main__": + main() diff --git a/hudi-integ-test/spark-pipelines/harness.py b/hudi-integ-test/spark-pipelines/harness.py new file mode 100644 index 000000000000..a7978b590c1e --- /dev/null +++ b/hudi-integ-test/spark-pipelines/harness.py @@ -0,0 +1,251 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Owned, foreground Spark Connect server and bounded SDP client processes (POSIX).""" + +import argparse +from collections import deque +import contextlib +import json +import os +from pathlib import Path +import signal +import socket +import subprocess +import sys +import tempfile +import time + + +SPARK_VERSION = "4.1.1" +HERE = Path(__file__).resolve().parent + + [email protected] +def owned_process(command, log, cwd, env): + """Kill only the new process group, including children left by an exited launcher.""" + with log.open("w") as output: + output.write("Command: " + repr([str(arg) for arg in command]) + "\n") + output.flush() + process = subprocess.Popen(command, cwd=cwd, env=env, stdout=output, + stderr=subprocess.STDOUT, start_new_session=True) + try: + yield process + finally: + stop_process_group(process) + + +def stop_process_group(process, grace=5): + # Retain the leader until its group has been signalled; never use pkill or Spark's global stop script. + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + process.wait() + return + deadline = time.monotonic() + grace + while time.monotonic() < deadline: + process.poll() + try: + os.killpg(process.pid, 0) + except ProcessLookupError: + break + time.sleep(0.1) + else: + with contextlib.suppress(ProcessLookupError): + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=grace) + + +def wait_for_process(process, timeout, log, server=None): + deadline = time.monotonic() + timeout + while process.poll() is None: + if server is not None and server.poll() is not None: + raise RuntimeError(f"Connect server exited with {server.returncode}; see server.log and {log}") + if time.monotonic() >= deadline: + raise TimeoutError(f"Client exceeded {timeout}s; see {log}") + time.sleep(0.1) + if process.returncode: + raise RuntimeError(f"Client exited with {process.returncode}; see {log}") + + +class ConnectHarness: + def __init__(self, bundle, logs, startup_timeout=90, client_timeout=180): + import pyspark + + if pyspark.__version__ != SPARK_VERSION: + raise ValueError(f"Install requirements.txt: expected PySpark {SPARK_VERSION}, got {pyspark.__version__}") + self.spark_home = Path(pyspark.__file__).resolve().parent + self.bundle = Path(bundle).resolve(strict=True) + if not self.bundle.is_file() or self.bundle.suffix != ".jar": + raise ValueError("--bundle must name the built Hudi Spark 4.1 bundle jar") + self.logs = Path(logs).resolve() + self.logs.mkdir(parents=True, exist_ok=True) + self.startup_timeout = startup_timeout + self.client_timeout = client_timeout + self.server = None + self.stack = contextlib.ExitStack() + + def __enter__(self): + try: + self.root = Path(self.stack.enter_context(tempfile.TemporaryDirectory(prefix="hudi-sdp-"))) + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + self.port = sock.getsockname()[1] + self.remote = f"sc://127.0.0.1:{self.port}" + self.env = self.process_environment() + conf = self.server_configuration() + (self.logs / "server-conf.json").write_text(json.dumps(conf, indent=2) + "\n") + command = [str(self.spark_home / "bin/spark-submit"), "--master", "local[2]", + "--driver-memory", "1g", "--jars", str(self.bundle)] + for key, value in conf.items(): + command.extend(["--conf", f"{key}={value}"]) + command.extend(["--class", "org.apache.spark.sql.connect.service.SparkConnectServer"]) + self.server = self.stack.enter_context(owned_process( + command, self.logs / "server.log", self.root, self.env)) + # A TCP check alone can accept an unrelated listener. The probe below also verifies our run ID. + self.wait_for_port() + self.client(HERE / "client.py", "probe", timeout=self.startup_timeout) + return self + except BaseException: + self.stack.close() + raise + + def __exit__(self, *exc): + self.stack.close() + + def process_environment(self): + env = os.environ.copy() + for key in ("SPARK_REMOTE", "SPARK_CONNECT_MODE_ENABLED", "SPARK_LOCAL_REMOTE", "SPARK_CONNECT_MODE", + "PYSPARK_SUBMIT_ARGS", "PYSPARK_GATEWAY_PORT", "PYSPARK_GATEWAY_SECRET", + "SPARK_SUBMIT_OPTS", "SPARK_JAVA_OPTS", "PYTHONPATH"): + env.pop(key, None) + conf_dir = self.root / "conf" + conf_dir.mkdir() + env.update(SPARK_HOME=str(self.spark_home), SPARK_CONF_DIR=str(conf_dir), + SPARK_LOCAL_IP="127.0.0.1", PYSPARK_PYTHON=sys.executable, + PYSPARK_DRIVER_PYTHON=sys.executable, PYTHONUNBUFFERED="1") + return env + + def server_configuration(self): + # Static/driver/storage configuration belongs here, never in a pipeline's session configuration. + return { + "spark.sql.extensions": "org.apache.spark.sql.hudi.HoodieSparkSessionExtension", + "spark.sql.catalog.spark_catalog": "org.apache.spark.sql.hudi.catalog.HoodieCatalog", + "spark.serializer": "org.apache.spark.serializer.KryoSerializer", + "spark.kryo.registrator": "org.apache.spark.HoodieSparkKryoRegistrar", + "spark.sql.catalogImplementation": "in-memory", + "spark.sql.warehouse.dir": (self.root / "warehouse").as_uri(), + "spark.hadoop.fs.defaultFS": "file:///", + "spark.local.dir": str(self.root / "spark-local"), + "spark.driver.host": "127.0.0.1", + "spark.driver.bindAddress": "127.0.0.1", + "spark.connect.grpc.binding.address": "127.0.0.1", + "spark.connect.grpc.binding.port": str(self.port), + "spark.port.maxRetries": "0", + "spark.ui.enabled": "false", + "spark.hudi.test.run.id": self.root.name, + } + + def wait_for_port(self): + deadline = time.monotonic() + self.startup_timeout + while time.monotonic() < deadline: + if self.server.poll() is not None: + raise RuntimeError(f"Connect server exited with {self.server.returncode}; see server.log") + try: + with socket.create_connection(("127.0.0.1", self.port), timeout=0.5): + return + except OSError: + time.sleep(0.2) + raise TimeoutError(f"Connect server startup exceeded {self.startup_timeout}s; see server.log") + + def run(self, command, name, timeout=None): + log = self.logs / f"{name}.log" + with owned_process(command, log, self.root, self.env) as process: + wait_for_process(process, timeout or self.client_timeout, log, self.server) + + def client(self, script, name, timeout=None): + self.run([sys.executable, str(script), name, self.remote, str(self.root)], name, timeout) + + def run_pipeline(self, spec, name="pipeline"): + self.run([str(self.spark_home / "bin/spark-pipelines"), "run", "--remote", self.remote, + "--spec", str(spec)], name) + + def smoke(self): + input_dir = self.root / "input" + input_dir.mkdir() + rows = [{"id": i, "ts": 100 + i, "value": f"value-{i}"} for i in range(1, 4)] + (input_dir / "events.json").write_text("".join(json.dumps(row) + "\n" for row in rows)) + spec = { + "name": "hudi_connect_smoke", + "storage": (self.root / "checkpoints").as_uri(), + "catalog": "spark_catalog", + "database": "sdp_test", + "libraries": [{"glob": {"include": "pipeline.py"}}], + "configuration": { + "spark.sql.shuffle.partitions": "2", + "spark.sql.pipelines.maxFlowRetryAttempts": "0", + "hudi.test.input": input_dir.as_uri(), + }, + } + (self.root / "pipeline.py").write_text((HERE / "pipeline.py").read_text()) + spec_path = self.root / "spark-pipeline.yml" + # JSON is a YAML subset; keep harness lifecycle tests independent of third-party Python packages. + spec_path.write_text(json.dumps(spec, indent=2) + "\n") + (self.logs / spec_path.name).write_text(spec_path.read_text()) + self.run_pipeline(spec_path) + self.client(HERE / "client.py", "verify") + if not list((self.root / "checkpoints").rglob("offsets/0")): + raise AssertionError("SDP did not create a streaming checkpoint under its isolated storage directory") + + +def positive_seconds(value): + seconds = int(value) + if seconds <= 0: + raise argparse.ArgumentTypeError("timeout must be positive") + return seconds + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--bundle", required=True, type=Path) + parser.add_argument("--logs", type=Path, default=Path("target/spark-pipelines-logs")) + parser.add_argument("--startup-timeout", type=positive_seconds, default=90) + parser.add_argument("--client-timeout", type=positive_seconds, default=180) + args = parser.parse_args() + args.logs.mkdir(parents=True, exist_ok=True) + logs = Path(tempfile.mkdtemp(prefix="run-", dir=args.logs.resolve())) + print(f"Spark Connect SDP logs: {logs}", flush=True) + # Let CI cancellation reach ExitStack, just like Ctrl-C; each child has its own process group. + def interrupted(signum, frame): + raise KeyboardInterrupt(f"Received signal {signum}") + signal.signal(signal.SIGTERM, interrupted) + try: + with ConnectHarness(args.bundle, logs, args.startup_timeout, args.client_timeout) as harness: + harness.smoke() + except (Exception, KeyboardInterrupt) as error: + print(f"FAILED: {error}", file=sys.stderr) + for log in sorted(logs.glob("*.log")): + print(f"\n--- {log.name} (last 60 lines) ---", file=sys.stderr) + with log.open(errors="replace") as output: + print("".join(deque(output, maxlen=60)), file=sys.stderr) + return 1 + print("PASS: spark-pipelines wrote the expected Hudi rows through Spark Connect; resources removed.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/hudi-integ-test/spark-pipelines/pipeline.py b/hudi-integ-test/spark-pipelines/pipeline.py new file mode 100644 index 000000000000..57926915b3bf --- /dev/null +++ b/hudi-integ-test/spark-pipelines/pipeline.py @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from pyspark import pipelines as dp +from pyspark.sql import SparkSession + +spark = SparkSession.active() + + [email protected](name="events", format="hudi", table_properties={ + "type": "cow", "primaryKey": "id", "preCombineField": "ts", +}, spark_conf={"spark.sql.shuffle.partitions": "2"}) +def events(): + return spark.readStream.schema("id INT, ts LONG, value STRING").json(spark.conf.get("hudi.test.input")) diff --git a/hudi-integ-test/spark-pipelines/requirements.txt b/hudi-integ-test/spark-pipelines/requirements.txt new file mode 100644 index 000000000000..f27c90ed6b76 --- /dev/null +++ b/hudi-integ-test/spark-pipelines/requirements.txt @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +pyspark[pipelines,connect]==4.1.1 diff --git a/hudi-integ-test/spark-pipelines/test_harness.py b/hudi-integ-test/spark-pipelines/test_harness.py new file mode 100644 index 000000000000..a2969594f51b --- /dev/null +++ b/hudi-integ-test/spark-pipelines/test_harness.py @@ -0,0 +1,98 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Fast lifecycle regressions; no Spark installation is needed for these tests.""" + +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import time +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +from harness import ConnectHarness, owned_process, stop_process_group, wait_for_process + + +class TestProcessLifecycle(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.log = self.root / "client.log" + + def process(self, code): + return owned_process([sys.executable, "-u", "-c", code], self.log, self.root, os.environ.copy()) + + def test_client_failure_keeps_diagnostics(self): + with self.process("print('intentional client failure'); raise SystemExit(7)") as process: + with self.assertRaisesRegex(RuntimeError, "exited with 7"): + wait_for_process(process, 5, self.log) + self.assertIn("intentional client failure", self.log.read_text()) + + def test_timeout_terminates_owned_process_without_stopping_another_server(self): + other = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"], start_new_session=True) + self.addCleanup(stop_process_group, other) + with self.assertRaises(TimeoutError): + with self.process("import time; time.sleep(60)") as process: + wait_for_process(process, 0.2, self.log) + self.assertIsNotNone(process.poll()) + self.assertIsNone(other.poll()) + + def test_server_exit_interrupts_client(self): + server = subprocess.Popen([sys.executable, "-c", "raise SystemExit(9)"]) + server.wait(timeout=5) + started = time.monotonic() + with self.assertRaisesRegex(RuntimeError, "server exited with 9"): + with self.process("import time; time.sleep(60)") as process: + wait_for_process(process, 30, self.log, server) + self.assertLess(time.monotonic() - started, 10) + self.assertIsNotNone(process.poll()) + + def test_kills_process_ignoring_term(self): + command = [sys.executable, "-u", "-c", + "import signal,time; signal.signal(signal.SIGTERM, signal.SIG_IGN); print('ready'); time.sleep(60)"] + process = subprocess.Popen(command, stdout=subprocess.PIPE, text=True, start_new_session=True) + self.addCleanup(stop_process_group, process, 0.2) + self.addCleanup(process.stdout.close) + self.assertEqual(process.stdout.readline().strip(), "ready") + stop_process_group(process, grace=0.2) + self.assertEqual(process.returncode, -9) + + def test_startup_failure_cleans_server_and_test_resources(self): + spark_home = self.root / "pyspark" + (spark_home / "bin").mkdir(parents=True) + submit = spark_home / "bin/spark-submit" + submit.write_text(f"#!{sys.executable}\nimport time\ntime.sleep(60)\n") + submit.chmod(0o755) + bundle = self.root / "bundle.jar" + bundle.touch() + fake_pyspark = SimpleNamespace(__version__="4.1.1", __file__=str(spark_home / "__init__.py")) + with patch.dict(sys.modules, pyspark=fake_pyspark): + harness = ConnectHarness(bundle, self.root / "logs", startup_timeout=0.2) + with self.assertRaisesRegex(TimeoutError, "startup exceeded"): + with harness: + self.fail("A server which never listens must not become ready") + self.assertFalse(harness.root.exists()) + self.assertIsNotNone(harness.server.poll()) + self.assertTrue((harness.logs / "server.log").is_file()) + + +if __name__ == "__main__": + unittest.main() diff --git a/hudi-spark-datasource/hudi-spark/pom.xml b/hudi-spark-datasource/hudi-spark/pom.xml index 8b3daf01fdfe..ca4a1fd4a29b 100644 --- a/hudi-spark-datasource/hudi-spark/pom.xml +++ b/hudi-spark-datasource/hudi-spark/pom.xml @@ -525,6 +525,54 @@ </dependencies> <profiles> + <!-- SDP is a Spark 4.1 API. Keep its server dependency and tests out of other profiles. --> + <profile> + <id>spark4.1</id> + <activation> + <property><name>spark4.1</name></property> + </activation> + <dependencies> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-connect_${scala.binary.version}</artifactId> + <version>${spark41.version}</version> + <scope>test</scope> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <executions> + <execution> + <id>add-sdp-test-source</id> + <phase>generate-test-sources</phase> + <goals><goal>add-test-source</goal></goals> + <configuration> + <sources><source>src/test/scala-spark4.1</source></sources> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.scalastyle</groupId> + <artifactId>scalastyle-maven-plugin</artifactId> + <executions> + <execution> + <id>check-sdp-test-style</id> + <phase>test-compile</phase> + <goals><goal>check</goal></goals> + <configuration> + <testSourceDirectory>${project.basedir}/src/test/scala-spark4.1</testSourceDirectory> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + <!-- Vortex requires Java 17 (vortex-jni/vortex-spark ship Java 17 bytecode). The Vortex round-trip test compiles/runs only under JDK 17; Java 11 builds skip it. --> <profile> diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala-spark4.1/org/apache/spark/sql/connect/hudi/TestHoodieSparkConnectPipelines.scala b/hudi-spark-datasource/hudi-spark/src/test/scala-spark4.1/org/apache/spark/sql/connect/hudi/TestHoodieSparkConnectPipelines.scala new file mode 100644 index 000000000000..5436d3efac97 --- /dev/null +++ b/hudi-spark-datasource/hudi-spark/src/test/scala-spark4.1/org/apache/spark/sql/connect/hudi/TestHoodieSparkConnectPipelines.scala @@ -0,0 +1,218 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.connect.hudi + +import org.apache.spark.sql.{Row, SparkSession} +import org.apache.spark.sql.connect.service.SparkConnectService +import org.apache.spark.util.Utils +import org.scalatest.{BeforeAndAfterAll, FunSuite} + +import java.io.{File, IOException} +import java.nio.charset.StandardCharsets.UTF_8 +import java.nio.file.Files +import java.util.UUID +import java.util.concurrent.TimeUnit + +/** Real pipelines CLI against a Connect service in the test JVM, without building a bundle. */ +class TestHoodieSparkConnectPipelines extends FunSuite with BeforeAndAfterAll { + private val python = sys.env.getOrElse("HUDI_SDP_PYTHON", "python3") + private val logs = new File("target/sdp-layer2", UUID.randomUUID().toString) + private var spark: SparkSession = _ + private var root: File = _ + private var started = false + + private lazy val pythonAvailable: Boolean = { + logs.mkdirs() + val probe = "import pyspark; from pyspark.pipelines import cli; " + + "from pyspark.sql.connect.session import SparkSession; " + + "assert pyspark.__version__ == '4.1.1', pyspark.__version__" + try { + runPython(Seq("-c", probe), "python-probe", 30)._1 == 0 + } catch { + case _: IOException => false + } + } + + private def startServer(): Unit = { + if (sys.env.get("HUDI_SDP_REQUIRE_PYTHON").contains("true")) { + assert(pythonAvailable, s"CI requires PySpark 4.1.1 pipelines/connect; see $logs") + } else { + assume(pythonAvailable, "Install the SDP requirements and set HUDI_SDP_PYTHON to enable this suite") + } + if (!started) { + root = Utils.createTempDir(namePrefix = "hudi-sdp-layer2") + spark = SparkSession.builder().master("local[2]").appName("Hudi SDP CLI tests") + .config("spark.ui.enabled", "false") + .config("spark.sql.warehouse.dir", new File(root, "warehouse").toURI.toString) + .config("spark.sql.catalogImplementation", "in-memory") + .config("spark.sql.extensions", "org.apache.spark.sql.hudi.HoodieSparkSessionExtension") + .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.hudi.catalog.HoodieCatalog") + .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") + .config("spark.kryo.registrator", "org.apache.spark.HoodieSparkKryoRegistrar") + .config("spark.connect.grpc.binding.address", "127.0.0.1") + .config("spark.connect.grpc.binding.port", "0") + .getOrCreate() + SparkConnectService.start(spark.sparkContext) + started = true + } + } + + test("CLI materialized view and streaming table write exact Hudi rows") { + withPipeline { (dir, database) => + write(dir, "flows.sql", + """CREATE MATERIALIZED VIEW mv USING hudi + |TBLPROPERTIES ('type'='cow', 'primaryKey'='id', 'preCombineField'='ts') + |AS SELECT 2 AS id, 2L AS ts, 'mv' AS value; + |CREATE STREAMING TABLE events USING hudi + |TBLPROPERTIES ('type'='cow', 'primaryKey'='id', 'preCombineField'='ts') + |AS SELECT * FROM STREAM input_events; + |""".stripMargin) + assertSuccess(runPipeline(dir, database)) + assert(rows(database, "mv") == Seq(Row(2, 2L, "mv"))) + assert(rows(database, "events") == Seq(Row(1, 1L, "old"))) + } + } + + test("flow-scoped SQL settings independently select insert and upsert") { + withPipeline { (dir, database) => + write(dir, "flows.sql", Seq("insert", "upsert").map { operation => + s"""SET spark.hoodie.datasource.write.operation=$operation; + |CREATE STREAMING TABLE ${operation}_events USING hudi + |TBLPROPERTIES ('type'='cow', 'primaryKey'='id', 'preCombineField'='ts') + |AS SELECT * FROM STREAM input_events; + |""".stripMargin + }.mkString("\n")) + assertSuccess(runPipeline(dir, database)) + assert(rows(database, "insert_events") == Seq.fill(2)(Row(1, 1L, "old"))) + assert(rows(database, "upsert_events") == Seq(Row(1, 1L, "old"))) + } + } + + test("a failing CLI flow reports the key error and leaves Connect usable") { + withPipeline { (dir, database) => + write(dir, "flows.sql", + """CREATE STREAMING TABLE bad_events USING hudi + |TBLPROPERTIES ('type'='cow', 'primaryKey'='id', 'preCombineField'='ts') + |AS SELECT CAST(NULL AS INT) AS id, ts, value FROM STREAM input_events; + |""".stripMargin) + val (exit, output) = runPipeline(dir, database) + assert(exit != 0, "The null-key flow unexpectedly succeeded") + assert(output.contains("recordKey") && output.contains("null"), output.takeRight(12000)) + val probe = "from pyspark.sql import SparkSession; " + + s"s = SparkSession.builder.remote('$remote').getOrCreate(); " + + "assert s.range(3).count() == 3; s.stop()" + assertSuccess(runPython(Seq("-c", probe), s"$database-after-failure", 60)) + } + } + + private def remote: String = s"sc://127.0.0.1:${SparkConnectService.localPort}" + + private def withPipeline(f: (File, String) => Unit): Unit = { + startServer() + val database = "sdp_" + UUID.randomUUID().toString.replace("-", "") + val dir = new File(root, database) + dir.mkdirs() + spark.sql(s"CREATE DATABASE $database").collect() + val input = new File(dir, "input") + input.mkdirs() + write(input, "events.json", """{"id":1,"ts":1,"value":"old"} + |{"id":1,"ts":1,"value":"old"} + |""".stripMargin) + write(dir, "source.py", + """from pyspark import pipelines as dp + |from pyspark.sql import SparkSession + |spark = SparkSession.active() + |@dp.temporary_view(name="input_events") + |def input_events(): + | return spark.readStream.schema("id INT, ts LONG, value STRING").json(spark.conf.get("hudi.test.input")) + |""".stripMargin) + try f(dir, database) finally { + spark.sql(s"DROP DATABASE $database CASCADE").collect() + } + } + + private def runPipeline(dir: File, database: String): (Int, String) = { + def quoted(value: String): String = "'" + value.replace("'", "''") + "'" + write(dir, "spark-pipeline.yml", + s"""name: $database + |catalog: spark_catalog + |database: $database + |storage: ${quoted(new File(dir, "checkpoints").toURI.toString)} + |libraries: + | - glob: {include: source.py} + | - glob: {include: flows.sql} + |configuration: + | hudi.test.input: ${quoted(new File(dir, "input").toURI.toString)} + | spark.sql.shuffle.partitions: '2' + | spark.sql.pipelines.maxFlowRetryAttempts: '0' + | spark.hoodie.metadata.enable: 'true' + |""".stripMargin) + Seq("source.py", "flows.sql", "spark-pipeline.yml").foreach { name => + Files.copy(new File(dir, name).toPath, new File(logs, s"$database-$name").toPath) + } + runPython(Seq("-m", "pyspark.pipelines.cli", "run", + "--spec", new File(dir, "spark-pipeline.yml").getAbsolutePath), database, 180) + } + + private def rows(database: String, table: String): Seq[Row] = { + spark.table(s"$database.$table").select("id", "ts", "value").orderBy("id", "ts").collect().toSeq + } + + private def write(dir: File, name: String, value: String): Unit = { + Files.write(new File(dir, name).toPath, value.getBytes(UTF_8)) + } + + private def assertSuccess(result: (Int, String)): Unit = { + assert(result._1 == 0, result._2.takeRight(12000)) + } + + private def runPython(args: Seq[String], name: String, timeout: Long): (Int, String) = { + val log = new File(logs, s"$name.log") + val builder = new ProcessBuilder((Seq(python) ++ args): _*) + .redirectErrorStream(true).redirectOutput(log) + Seq("SPARK_REMOTE", "SPARK_CONNECT_MODE_ENABLED", "SPARK_LOCAL_REMOTE", "SPARK_CONNECT_MODE", + "PYSPARK_GATEWAY_PORT", "PYSPARK_GATEWAY_SECRET", "PYTHONPATH").foreach(builder.environment().remove) + builder.environment().put("PYTHONUNBUFFERED", "1") + // The spark-pipelines launcher translates --remote into SPARK_REMOTE before invoking cli.py. + if (started) builder.environment().put("SPARK_REMOTE", remote) + val process = builder.start() + try { + assert(process.waitFor(timeout, TimeUnit.SECONDS), s"CLI timed out after ${timeout}s; see $log") + (process.exitValue(), new String(Files.readAllBytes(log.toPath), UTF_8)) + } finally { + process.destroy() + if (!process.waitFor(5, TimeUnit.SECONDS)) { + process.destroyForcibly() + process.waitFor(5, TimeUnit.SECONDS) + } + } + } + + override protected def afterAll(): Unit = { + try { + if (spark != null) { + try SparkConnectService.stop() finally spark.stop() + } + } finally { + if (root != null) Utils.deleteRecursively(root) + SparkSession.clearActiveSession() + SparkSession.clearDefaultSession() + super.afterAll() + } + } +}
