This is an automated email from the ASF dual-hosted git repository.
shahar1 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 1e0e003cea8 Template conn_id and config_kwargs in @task.pyspark
(#75764)
1e0e003cea8 is described below
commit 1e0e003cea8d1c9989697777af23bb2336759715
Author: Sebastian Daberdaku <[email protected]>
AuthorDate: Fri Nov 6 10:59:21 2026 +0100
Template conn_id and config_kwargs in @task.pyspark (#75764)
_PySparkDecoratedOperator inherited DecoratedOperator.template_fields
through its MRO, so Jinja in config_kwargs and conn_id was passed to
Spark as literal text, while PySparkOperator renders both. Declare the
union of the decorator fields and the two Spark-specific ones, and say
in the decorator guide that both parameters accept Jinja.
closes: #75762
---
providers/apache/spark/docs/decorators/pyspark.rst | 4 ++++
.../providers/apache/spark/decorators/pyspark.py | 2 ++
.../unit/apache/spark/decorators/test_pyspark.py | 22 ++++++++++++++++++++++
3 files changed, 28 insertions(+)
diff --git a/providers/apache/spark/docs/decorators/pyspark.rst
b/providers/apache/spark/docs/decorators/pyspark.rst
index c83b5980cbd..1962445bc36 100644
--- a/providers/apache/spark/docs/decorators/pyspark.rst
+++ b/providers/apache/spark/docs/decorators/pyspark.rst
@@ -37,6 +37,10 @@ config_kwargs: dict
The kwargs used for initializing the SparkConf object. This overrides
the spark configuration options set in the connection.
+You can use :ref:`Jinja templating <concepts:jinja-templating>` with the
``conn_id`` and
+``config_kwargs`` parameters, which allows you to dynamically determine
values, for example
+``config_kwargs={"spark.app.name": "app-{{ ds }}"}``.
+
Example
-------
diff --git
a/providers/apache/spark/src/airflow/providers/apache/spark/decorators/pyspark.py
b/providers/apache/spark/src/airflow/providers/apache/spark/decorators/pyspark.py
index 0aac4fc94f6..86ba28d0a12 100644
---
a/providers/apache/spark/src/airflow/providers/apache/spark/decorators/pyspark.py
+++
b/providers/apache/spark/src/airflow/providers/apache/spark/decorators/pyspark.py
@@ -31,6 +31,8 @@ from airflow.providers.common.compat.sdk import (
class _PySparkDecoratedOperator(DecoratedOperator, PySparkOperator):
custom_operator_name = "@task.pyspark"
+ template_fields: Sequence[str] = (*DecoratedOperator.template_fields,
"conn_id", "config_kwargs")
+
def __init__(
self,
*,
diff --git
a/providers/apache/spark/tests/unit/apache/spark/decorators/test_pyspark.py
b/providers/apache/spark/tests/unit/apache/spark/decorators/test_pyspark.py
index f4bd4fd33f7..3bb37af2c6b 100644
--- a/providers/apache/spark/tests/unit/apache/spark/decorators/test_pyspark.py
+++ b/providers/apache/spark/tests/unit/apache/spark/decorators/test_pyspark.py
@@ -118,6 +118,28 @@ class TestPysparkDecorator:
spark_mock.builder.config.assert_called_once_with(conf=conf_mock())
+ @pytest.mark.db_test
+ @mock.patch("pyspark.SparkConf")
+ @mock.patch("pyspark.sql.SparkSession")
+ def test_pyspark_decorator_renders_template_fields(self, spark_mock,
conf_mock, dag_maker):
+ config = FakeConfig()
+ conf_mock.return_value = config
+
+ @task.pyspark(
+ conn_id="spark-{{ 'connect' }}",
+ config_kwargs={"spark.app.name": "app-{{ task.task_id }}"},
+ )
+ def f(spark):
+ return True
+
+ with dag_maker():
+ f()
+
+ ti = dag_maker.run_ti("f")
+ assert ti.xcom_pull()
+ assert config.get("spark.remote") == "sc://localhost"
+ assert config.get("spark.app.name") == "app-f"
+
@pytest.mark.db_test
@mock.patch("pyspark.SparkConf")
@mock.patch("pyspark.sql.SparkSession")