This is an automated email from the ASF dual-hosted git repository.

asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 400b870c2a42 [SPARK-60188][PYTHON] Warn on unknown transpiler names in 
pyTranspilers
400b870c2a42 is described below

commit 400b870c2a42e41105c41049de64823059d5f175
Author: Guan-Ming Chiu <[email protected]>
AuthorDate: Fri Oct 30 17:37:57 2026 -0700

    [SPARK-60188][PYTHON] Warn on unknown transpiler names in pyTranspilers
    
    ### What changes were proposed in this pull request?
    
    Warn when `spark.sql.experimental.optimizer.pyTranspilers` names a 
transpiler that is not registered.
    
    ### Why are the changes needed?
    
    `_get_transpilers` silently drops unknown names, so a typo turns 
transpilation off without any signal.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes, an unknown name in this experimental conf now emits a `UserWarning`.
    
    ### How was this patch tested?
    
    New unit test in `test_udf_transpile_unit.py`.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Code (Opus 5.5)
    
    Closes #59580 from guan404ming/SPARK-60188-warn-unknown-transpilers.
    
    Authored-by: Guan-Ming Chiu <[email protected]>
    Signed-off-by: Holden Karau <[email protected]>
---
 .../pyspark/sql/tests/test_udf_transpile_unit.py   | 22 ++++++++++++++++++++++
 python/pyspark/sql/transpile.py                    |  7 +++++++
 2 files changed, 29 insertions(+)

diff --git a/python/pyspark/sql/tests/test_udf_transpile_unit.py 
b/python/pyspark/sql/tests/test_udf_transpile_unit.py
index c19c38814fff..0df1f4b969fb 100644
--- a/python/pyspark/sql/tests/test_udf_transpile_unit.py
+++ b/python/pyspark/sql/tests/test_udf_transpile_unit.py
@@ -27,6 +27,7 @@ property-based suite lives in 
``test_udf_transpile_hypothesis.py``.
 import unittest
 
 from pyspark.sql import Row
+from pyspark.sql.transpile import CatalystTranspiler, _get_transpilers
 from pyspark.sql.types import (
     BinaryType,
     BooleanType,
@@ -179,6 +180,27 @@ class UDFTranspileUnitTests(ReusedSQLTestCase):
                 "when both transpilePyUDFs and ANSI mode are enabled",
             )
 
+    def test_get_transpilers_warns_on_unknown_names(self):
+        import warnings
+
+        for names, expected, unknown in [
+            ("catalyst", [CatalystTranspiler], None),
+            ("catalyst,catalist", [CatalystTranspiler], "catalist"),
+            ("catalist", [], "catalist"),
+        ]:
+            with self.subTest(names=names):
+                with 
self.sql_conf({"spark.sql.experimental.optimizer.pyTranspilers": names}):
+                    with warnings.catch_warnings(record=True) as caught:
+                        warnings.simplefilter("always")
+                        transpilers = _get_transpilers(self.spark)
+                self.assertEqual([type(t) for t in transpilers], expected)
+                messages = [str(w.message) for w in caught if "pyTranspilers" 
in str(w.message)]
+                if unknown is None:
+                    self.assertEqual(messages, [])
+                else:
+                    self.assertEqual(len(messages), 1)
+                    self.assertIn(repr(unknown), messages[0])
+
     def test_udf_transpile_does_not_mask_char_varchar_ddl(self):
         # DDL CHAR/VARCHAR is parsed via ``returnType`` inside the optional
         # transpilation try. That structured error must not become a
diff --git a/python/pyspark/sql/transpile.py b/python/pyspark/sql/transpile.py
index bef13efe9a7d..02c2ba9a25bd 100644
--- a/python/pyspark/sql/transpile.py
+++ b/python/pyspark/sql/transpile.py
@@ -1108,6 +1108,13 @@ def _get_transpilers(session: "SparkSession") -> 
List[AbstractTranspiler]:
     if not configured_transpilers:
         return []
     transpiler_names = configured_transpilers.split(",")
+    unknown_names = [name for name in transpiler_names if name not in 
AbstractTranspiler.varieties]
+    if unknown_names:
+        warnings.warn(
+            f"Ignoring unknown transpilers {unknown_names} in "
+            "spark.sql.experimental.optimizer.pyTranspilers; registered 
transpilers are "
+            f"{sorted(AbstractTranspiler.varieties)}"
+        )
     return [
         AbstractTranspiler.varieties[name]()
         for name in transpiler_names


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to