This is an automated email from the ASF dual-hosted git repository.

shahar1 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new 1e0e003cea8 Template conn_id and config_kwargs in @task.pyspark 
(#75764)
1e0e003cea8 is described below

commit 1e0e003cea8d1c9989697777af23bb2336759715
Author: Sebastian Daberdaku <[email protected]>
AuthorDate: Fri Nov 6 10:59:21 2026 +0100

    Template conn_id and config_kwargs in @task.pyspark (#75764)
    
    _PySparkDecoratedOperator inherited DecoratedOperator.template_fields
    through its MRO, so Jinja in config_kwargs and conn_id was passed to
    Spark as literal text, while PySparkOperator renders both. Declare the
    union of the decorator fields and the two Spark-specific ones, and say
    in the decorator guide that both parameters accept Jinja.
    
    closes: #75762
---
 providers/apache/spark/docs/decorators/pyspark.rst |  4 ++++
 .../providers/apache/spark/decorators/pyspark.py   |  2 ++
 .../unit/apache/spark/decorators/test_pyspark.py   | 22 ++++++++++++++++++++++
 3 files changed, 28 insertions(+)

diff --git a/providers/apache/spark/docs/decorators/pyspark.rst 
b/providers/apache/spark/docs/decorators/pyspark.rst
index c83b5980cbd..1962445bc36 100644
--- a/providers/apache/spark/docs/decorators/pyspark.rst
+++ b/providers/apache/spark/docs/decorators/pyspark.rst
@@ -37,6 +37,10 @@ config_kwargs: dict
     The kwargs used for initializing the SparkConf object. This overrides
     the spark configuration options set in the connection.
 
+You can use :ref:`Jinja templating <concepts:jinja-templating>` with the 
``conn_id`` and
+``config_kwargs`` parameters, which allows you to dynamically determine 
values, for example
+``config_kwargs={"spark.app.name": "app-{{ ds }}"}``.
+
 
 Example
 -------
diff --git 
a/providers/apache/spark/src/airflow/providers/apache/spark/decorators/pyspark.py
 
b/providers/apache/spark/src/airflow/providers/apache/spark/decorators/pyspark.py
index 0aac4fc94f6..86ba28d0a12 100644
--- 
a/providers/apache/spark/src/airflow/providers/apache/spark/decorators/pyspark.py
+++ 
b/providers/apache/spark/src/airflow/providers/apache/spark/decorators/pyspark.py
@@ -31,6 +31,8 @@ from airflow.providers.common.compat.sdk import (
 class _PySparkDecoratedOperator(DecoratedOperator, PySparkOperator):
     custom_operator_name = "@task.pyspark"
 
+    template_fields: Sequence[str] = (*DecoratedOperator.template_fields, 
"conn_id", "config_kwargs")
+
     def __init__(
         self,
         *,
diff --git 
a/providers/apache/spark/tests/unit/apache/spark/decorators/test_pyspark.py 
b/providers/apache/spark/tests/unit/apache/spark/decorators/test_pyspark.py
index f4bd4fd33f7..3bb37af2c6b 100644
--- a/providers/apache/spark/tests/unit/apache/spark/decorators/test_pyspark.py
+++ b/providers/apache/spark/tests/unit/apache/spark/decorators/test_pyspark.py
@@ -118,6 +118,28 @@ class TestPysparkDecorator:
 
         spark_mock.builder.config.assert_called_once_with(conf=conf_mock())
 
+    @pytest.mark.db_test
+    @mock.patch("pyspark.SparkConf")
+    @mock.patch("pyspark.sql.SparkSession")
+    def test_pyspark_decorator_renders_template_fields(self, spark_mock, 
conf_mock, dag_maker):
+        config = FakeConfig()
+        conf_mock.return_value = config
+
+        @task.pyspark(
+            conn_id="spark-{{ 'connect' }}",
+            config_kwargs={"spark.app.name": "app-{{ task.task_id }}"},
+        )
+        def f(spark):
+            return True
+
+        with dag_maker():
+            f()
+
+        ti = dag_maker.run_ti("f")
+        assert ti.xcom_pull()
+        assert config.get("spark.remote") == "sc://localhost"
+        assert config.get("spark.app.name") == "app-f"
+
     @pytest.mark.db_test
     @mock.patch("pyspark.SparkConf")
     @mock.patch("pyspark.sql.SparkSession")

Reply via email to