This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new dd756d8 [SPARK-60955][CONNECT] Vendor pyspark.ml.connect.functions so
that vector_to_array and array_to_vector work
dd756d8 is described below
commit dd756d84fdab1a8e84a6170b8a481dcc74eb2e33
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Fri Nov 6 13:12:34 2026 +0900
[SPARK-60955][CONNECT] Vendor pyspark.ml.connect.functions so that
vector_to_array and array_to_vector work
### What changes were proposed in this pull request?
[SPARK-60955](https://issues.apache.org/jira/browse/SPARK-60955):
`pyspark.ml.connect.functions` is now vendored, byte for byte from Apache Spark
v4.3.0. It builds the columns of `pyspark.ml.functions.vector_to_array` and
`array_to_vector` through `pyspark.sql.connect.functions.builtin`, which the
client provides. `scripts/check_vendored_upstream.sh` now checks it against the
tag.
### Why are the changes needed?
`pyspark.ml.functions` (vendored with 90fcaf7) imports these functions from
`pyspark.ml.connect.functions` under Spark Connect, which the client did not
have. So every call raised `ImportError: cannot import name 'functions' from
'pyspark.ml.connect'`. For example, `spark.sql("SELECT array(1.5D, 2.0D)
a").select(vector_to_array(array_to_vector("a")))` failed, where PySpark 4.3.0
gives `[1.5, 2.0]`.
### Does this PR introduce _any_ user-facing change?
Yes, `vector_to_array` and `array_to_vector` work, as in PySpark 4.3.0.
### How was this patch tested?
- Against a local Spark 4.3.0 server, the reprs of `vector_to_array` with
both dtypes and of `array_to_vector`, and their values, are the same as with
official PySpark 4.3.0.
- `scripts/check_vendored_upstream.sh` reports the new file as matching
v4.3.0.
- A new offline test checks the reprs. A new live test checks the values,
and it also passes with official PySpark 4.3.0.
- The offline Python suites (614 passed, 8 live cases skipped), the live
drop-in and Arrow suites (58 passed) and live cases (10 passed). The change has
no Rust code.
Collecting a vector column still gives a `Row` rather than a `DenseVector`,
and `dtypes` shows `udt` rather than `vector`, as values of a user-defined type
are not deserialized. That is filed separately.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #175 from HyukjinKwon/SPARK-60955-ml-functions.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
python/pyspark/ml/connect/functions.py | 91 ++++++++++++++++++++++++++++++++++
python/tests/test_dropin_e2e.py | 8 +++
python/tests/test_dropin_offline.py | 9 ++++
scripts/check_vendored_upstream.sh | 3 ++
4 files changed, 111 insertions(+)
diff --git a/python/pyspark/ml/connect/functions.py
b/python/pyspark/ml/connect/functions.py
new file mode 100644
index 0000000..5f2738c
--- /dev/null
+++ b/python/pyspark/ml/connect/functions.py
@@ -0,0 +1,91 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from typing import Any, TYPE_CHECKING
+
+from pyspark.ml import functions as PyMLFunctions
+from pyspark.sql.column import Column
+
+if TYPE_CHECKING:
+ from pyspark.sql._typing import UserDefinedFunctionLike
+
+
+def vector_to_array(col: Column, dtype: str = "float64") -> Column:
+ from pyspark.sql.connect.functions.builtin import _invoke_function,
_to_col, lit
+
+ return _invoke_function("vector_to_array", _to_col(col), lit(dtype))
+
+
+vector_to_array.__doc__ = PyMLFunctions.vector_to_array.__doc__
+
+
+def array_to_vector(col: Column) -> Column:
+ from pyspark.sql.connect.functions.builtin import _invoke_function, _to_col
+
+ return _invoke_function("array_to_vector", _to_col(col))
+
+
+array_to_vector.__doc__ = PyMLFunctions.array_to_vector.__doc__
+
+
+def predict_batch_udf(*args: Any, **kwargs: Any) -> "UserDefinedFunctionLike":
+ return PyMLFunctions.predict_batch_udf(*args, **kwargs)
+
+
+predict_batch_udf.__doc__ = PyMLFunctions.predict_batch_udf.__doc__
+
+
+def _test() -> None:
+ import os
+ import sys
+
+ if os.environ.get("PYTHON_GIL", "?") == "0":
+ print("Not supported in no-GIL mode", file=sys.stderr)
+ sys.exit(0)
+
+ from pyspark.testing.utils import should_test_connect
+
+ if not should_test_connect:
+ print("Skipping pyspark.ml.connect.functions doctests",
file=sys.stderr)
+ sys.exit(0)
+
+ import doctest
+ from pyspark.sql import SparkSession as PySparkSession
+ import pyspark.ml.connect.functions
+
+ globs = pyspark.ml.connect.functions.__dict__.copy()
+
+ globs["spark"] = (
+ PySparkSession.builder.appName("ml.connect.functions tests")
+ .remote(os.environ.get("SPARK_CONNECT_TESTING_REMOTE", "local[4]"))
+ .getOrCreate()
+ )
+
+ failure_count, test_count = doctest.testmod(
+ pyspark.ml.connect.functions,
+ globs=globs,
+ optionflags=doctest.ELLIPSIS
+ | doctest.NORMALIZE_WHITESPACE
+ | doctest.IGNORE_EXCEPTION_DETAIL,
+ )
+ globs["spark"].stop()
+ if failure_count:
+ sys.exit(-1)
+
+
+if __name__ == "__main__":
+ _test()
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index a245b21..9734e93 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -577,6 +577,14 @@ def
test_maps_keep_their_entries_in_order_with_keys_of_their_type(session):
assert list(df.first()[0].items()) == [(2, "b"), (1, "a")]
+def test_ml_vector_functions_as_pyspark(session):
+ from pyspark.ml.functions import array_to_vector, vector_to_array
+
+ df = session.sql("SELECT array(1.5D, 2.0D) AS
a").select(array_to_vector("a").alias("vec"))
+ row = df.select(vector_to_array("vec").alias("d"), vector_to_array("vec",
"float32").alias("f"))
+ assert tuple(row.first()) == ([1.5, 2.0], [1.5, 2.0])
+
+
def test_create_dataframe_merges_the_types_of_every_row_as_pyspark(session):
import array
import decimal
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 23747e5..df2d96f 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -3500,6 +3500,15 @@ def test_session_hook_is_present_and_passthrough():
assert hook.on_execute_plan(sentinel) is sentinel
+def test_ml_vector_functions_build_their_columns_as_pyspark():
+ from pyspark.ml.functions import array_to_vector, vector_to_array
+
+ # Through pyspark.ml.connect.functions, as in PySpark Connect.
+ assert repr(vector_to_array("v")) == "Column<'vector_to_array(v,
float64)'>"
+ assert repr(vector_to_array("v", "float32")) ==
"Column<'vector_to_array(v, float32)'>"
+ assert repr(array_to_vector("a")) == "Column<'array_to_vector(a)'>"
+
+
def test_ml_connect_base_classes_and_functions():
from pyspark.ml.connect import Estimator, Transformer, Evaluator, Model,
PipelineModel
diff --git a/scripts/check_vendored_upstream.sh
b/scripts/check_vendored_upstream.sh
index 2de216e..d837858 100644
--- a/scripts/check_vendored_upstream.sh
+++ b/scripts/check_vendored_upstream.sh
@@ -90,6 +90,9 @@ VENDORED_FILES=(
"ml/functions.py"
# ML connect base ABCs (Estimator/Transformer/Evaluator/Model) - pure
upstream.
"ml/connect/base.py"
+ # vector_to_array/array_to_vector of pyspark.ml.functions build their
columns here, through
+ # pyspark.sql.connect.functions.builtin - pure upstream.
+ "ml/connect/functions.py"
# ML: param (param API is unmodified)
"ml/param/__init__.py"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]