This is an automated email from the ASF dual-hosted git repository.
eladkal pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 58bffa6862 Rename `SparkSqlOperator`'s field's name to comply with
templated fields validation (#38045)
58bffa6862 is described below
commit 58bffa686238102cb628f19a16d9ad1c65ecda15
Author: Shahar Epstein <[email protected]>
AuthorDate: Tue Mar 12 10:33:28 2024 +0200
Rename `SparkSqlOperator`'s field's name to comply with templated fields
validation (#38045)
---
.pre-commit-config.yaml | 1 -
airflow/providers/apache/spark/operators/spark_sql.py | 18 +++++++++++++++---
2 files changed, 15 insertions(+), 4 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 68d7c688a9..ba2707de50 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -340,7 +340,6 @@ repos:
^airflow\/providers\/apache\/spark\/operators\/spark_submit.py\.py$|
^airflow\/providers\/google\/cloud\/operators\/vertex_ai\/auto_ml\.py$|
^airflow\/providers\/apache\/spark\/operators\/spark_submit\.py$|
- ^airflow\/providers\/apache\/spark\/operators\/spark_sql\.py$|
^airflow\/providers\/databricks\/operators\/databricks_sql\.py$|
)$
- id: ruff
diff --git a/airflow/providers/apache/spark/operators/spark_sql.py
b/airflow/providers/apache/spark/operators/spark_sql.py
index 47487fa384..7e48b748c6 100644
--- a/airflow/providers/apache/spark/operators/spark_sql.py
+++ b/airflow/providers/apache/spark/operators/spark_sql.py
@@ -19,6 +19,9 @@ from __future__ import annotations
from typing import TYPE_CHECKING, Any, Sequence
+from deprecated import deprecated
+
+from airflow.exceptions import AirflowProviderDeprecationWarning
from airflow.models import BaseOperator
from airflow.providers.apache.spark.hooks.spark_sql import SparkSqlHook
@@ -52,9 +55,9 @@ class SparkSqlOperator(BaseOperator):
(Default: The ``queue`` value set in the Connection, or ``"default"``)
"""
- template_fields: Sequence[str] = ("_sql",)
+ template_fields: Sequence[str] = ("sql",)
template_ext: Sequence[str] = (".sql", ".hql")
- template_fields_renderers = {"_sql": "sql"}
+ template_fields_renderers = {"sql": "sql"}
def __init__(
self,
@@ -75,7 +78,7 @@ class SparkSqlOperator(BaseOperator):
**kwargs: Any,
) -> None:
super().__init__(**kwargs)
- self._sql = sql
+ self.sql = sql
self._conf = conf
self._conn_id = conn_id
self._total_executor_cores = total_executor_cores
@@ -90,6 +93,15 @@ class SparkSqlOperator(BaseOperator):
self._yarn_queue = yarn_queue
self._hook: SparkSqlHook | None = None
+ @property
+ @deprecated(
+ reason="`_sql` is deprecated and will be removed in the future. Please
use `sql` instead.",
+ category=AirflowProviderDeprecationWarning,
+ )
+ def _sql(self):
+ """Alias for ``sql``, used for compatibility (deprecated)."""
+ return self.sql
+
def execute(self, context: Context) -> None:
"""Call the SparkSqlHook to run the provided sql query."""
if self._hook is None:
