This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 400b870c2a42 [SPARK-60188][PYTHON] Warn on unknown transpiler names in
pyTranspilers
400b870c2a42 is described below
commit 400b870c2a42e41105c41049de64823059d5f175
Author: Guan-Ming Chiu <[email protected]>
AuthorDate: Fri Oct 30 17:37:57 2026 -0700
[SPARK-60188][PYTHON] Warn on unknown transpiler names in pyTranspilers
### What changes were proposed in this pull request?
Warn when `spark.sql.experimental.optimizer.pyTranspilers` names a
transpiler that is not registered.
### Why are the changes needed?
`_get_transpilers` silently drops unknown names, so a typo turns
transpilation off without any signal.
### Does this PR introduce _any_ user-facing change?
Yes, an unknown name in this experimental conf now emits a `UserWarning`.
### How was this patch tested?
New unit test in `test_udf_transpile_unit.py`.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Code (Opus 5.5)
Closes #59580 from guan404ming/SPARK-60188-warn-unknown-transpilers.
Authored-by: Guan-Ming Chiu <[email protected]>
Signed-off-by: Holden Karau <[email protected]>
---
.../pyspark/sql/tests/test_udf_transpile_unit.py | 22 ++++++++++++++++++++++
python/pyspark/sql/transpile.py | 7 +++++++
2 files changed, 29 insertions(+)
diff --git a/python/pyspark/sql/tests/test_udf_transpile_unit.py
b/python/pyspark/sql/tests/test_udf_transpile_unit.py
index c19c38814fff..0df1f4b969fb 100644
--- a/python/pyspark/sql/tests/test_udf_transpile_unit.py
+++ b/python/pyspark/sql/tests/test_udf_transpile_unit.py
@@ -27,6 +27,7 @@ property-based suite lives in
``test_udf_transpile_hypothesis.py``.
import unittest
from pyspark.sql import Row
+from pyspark.sql.transpile import CatalystTranspiler, _get_transpilers
from pyspark.sql.types import (
BinaryType,
BooleanType,
@@ -179,6 +180,27 @@ class UDFTranspileUnitTests(ReusedSQLTestCase):
"when both transpilePyUDFs and ANSI mode are enabled",
)
+ def test_get_transpilers_warns_on_unknown_names(self):
+ import warnings
+
+ for names, expected, unknown in [
+ ("catalyst", [CatalystTranspiler], None),
+ ("catalyst,catalist", [CatalystTranspiler], "catalist"),
+ ("catalist", [], "catalist"),
+ ]:
+ with self.subTest(names=names):
+ with
self.sql_conf({"spark.sql.experimental.optimizer.pyTranspilers": names}):
+ with warnings.catch_warnings(record=True) as caught:
+ warnings.simplefilter("always")
+ transpilers = _get_transpilers(self.spark)
+ self.assertEqual([type(t) for t in transpilers], expected)
+ messages = [str(w.message) for w in caught if "pyTranspilers"
in str(w.message)]
+ if unknown is None:
+ self.assertEqual(messages, [])
+ else:
+ self.assertEqual(len(messages), 1)
+ self.assertIn(repr(unknown), messages[0])
+
def test_udf_transpile_does_not_mask_char_varchar_ddl(self):
# DDL CHAR/VARCHAR is parsed via ``returnType`` inside the optional
# transpilation try. That structured error must not become a
diff --git a/python/pyspark/sql/transpile.py b/python/pyspark/sql/transpile.py
index bef13efe9a7d..02c2ba9a25bd 100644
--- a/python/pyspark/sql/transpile.py
+++ b/python/pyspark/sql/transpile.py
@@ -1108,6 +1108,13 @@ def _get_transpilers(session: "SparkSession") ->
List[AbstractTranspiler]:
if not configured_transpilers:
return []
transpiler_names = configured_transpilers.split(",")
+ unknown_names = [name for name in transpiler_names if name not in
AbstractTranspiler.varieties]
+ if unknown_names:
+ warnings.warn(
+ f"Ignoring unknown transpilers {unknown_names} in "
+ "spark.sql.experimental.optimizer.pyTranspilers; registered
transpilers are "
+ f"{sorted(AbstractTranspiler.varieties)}"
+ )
return [
AbstractTranspiler.varieties[name]()
for name in transpiler_names
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]