This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 9d9b0e7  [SPARK-61065][CONNECT][PYTHON] Export the names of PySpark 
from pyspark.sql.functions, and type the literals of NumPy scalars by their 
dtype, as PySpark does
9d9b0e7 is described below

commit 9d9b0e7775f818a79ed2f88bf1d977b0bfddfbb9
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 11:42:24 2026 +0900

    [SPARK-61065][CONNECT][PYTHON] Export the names of PySpark from 
pyspark.sql.functions, and type the literals of NumPy scalars by their dtype, 
as PySpark does
    
    ### What changes were proposed in this pull request?
    
    - `pyspark.sql.functions.__all__` is now official PySpark 4.3.0's static 
list of 562 names, in its order, so `from pyspark.sql.functions import *` no 
longer exports `chr`, `random`, `uuid`, `countDistinct`, `bitwiseNOT`, 
`shiftLeft`, `shiftRight`, `shiftRightUnsigned`, `toDegrees`, `toRadians`, 
`sumDistinct`, `approxCountDistinct` and the avro and protobuf functions; those 
that official has stay attributes. The public attributes of the module are 
official's: `from_avro`, `to_avro`, `f [...]
    - `lit()`, and Column operations with a value, type a NumPy scalar by its 
dtype, as official's `_from_numpy_type` does: `int8`/`int16`/`int32`/`int64` as 
tinyint/smallint/int/bigint, `float32` as float, `float64` as double, `bool_` 
as boolean; unsigned ints, `float16`, `longdouble` and complex numbers raise 
`UNSUPPORTED_DATA_TYPE` (`crates/pyspark-rs/src/functions.rs`).
    
    ### Why are the changes needed?
    
    [SPARK-61065](https://issues.apache.org/jira/browse/SPARK-61065): the star 
import, common in notebooks, shadowed builtins:
    
    | Code | Current master | This change and PySpark 4.3.0 |
    | --- | --- | --- |
    | `from pyspark.sql.functions import *; chr(65)` | `PySparkTypeError 
[NOT_EXPECTED_TYPE]` | `'A'` |
    | `import random; from pyspark.sql.functions import *; random.randint(1, 
1)` | `AttributeError` | `1` |
    | `hasattr(F, "from_avro")` | `True` | `False` |
    | `spark.range(1).select(F.lit(np.int64(5))).schema` | `struct<5:int>` | 
`struct<5:bigint>` |
    | `F.lit(np.float32(1.5))` | double | float |
    | `F.lit(np.uint8(5))` | int | `UNSUPPORTED_DATA_TYPE` `uint8` |
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. The star import and the module's attributes are official's, the avro 
and protobuf functions are on their official modules, and NumPy scalars are 
typed as in PySpark.
    
    ### How was this patch tested?
    
    - New offline tests in `python/tests/test_dropin_offline.py` for the 
exports, the avro and protobuf modules, and the literal types of each NumPy 
scalar (from the plan), and a live test in `python/tests/test_dropin_e2e.py`; 
`test_functions_exports_fresh_process` now compares with official's list.
    - The offline suite (961 passed, 8 skipped); about 40 `lit` cases match 
official's types, column names and errors.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #256 from HyukjinKwon/SPARK-61065-functions-exports.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/functions.rs       |  52 +++
 python/pyspark/sql/avro/functions.py     |  66 ++-
 python/pyspark/sql/functions.py          | 727 +++++++++++++++++++++++++------
 python/pyspark/sql/protobuf/functions.py |  85 +++-
 python/tests/test_dropin_e2e.py          |   9 +
 python/tests/test_dropin_offline.py      | 239 +++++++++-
 6 files changed, 1035 insertions(+), 143 deletions(-)

diff --git a/crates/pyspark-rs/src/functions.rs 
b/crates/pyspark-rs/src/functions.rs
index 31e1aca..3a2c0ed 100644
--- a/crates/pyspark-rs/src/functions.rs
+++ b/crates/pyspark-rs/src/functions.rs
@@ -224,6 +224,11 @@ pub fn to_column(obj: &Bound<'_, PyAny>) -> 
PyResult<Column> {
         ));
     }
 
+    // A NumPy scalar is typed by its dtype, as PySpark's 
`LiteralExpression._infer_type`.
+    if let Some(lit) = numpy_scalar_literal(obj)? {
+        return Ok(Column::new(Expression::literal(lit)));
+    }
+
     // Non-primitive scalars must be checked BEFORE int/float: Decimal defines
     // __float__/__int__, and a pandas Timestamp (a datetime subclass) defines 
__int__
     // (nanoseconds), so an early int/float extract would silently mis-encode 
them.
@@ -288,6 +293,53 @@ fn numpy_ndarray_ndim(obj: &Bound<'_, PyAny>) -> 
PyResult<Option<usize>> {
     }
 }
 
+/// The literal of a NumPy scalar, typed as PySpark's 
`LiteralExpression._infer_type` types it: a
+/// bool, a signed int of 8 to 64 bits (tinyint to bigint), or a float32 or 
float64. A str or bytes
+/// scalar is `None`, for the string and binary conversions to take. Any other 
NumPy scalar (an
+/// unsigned int, float16, longdouble, complex, datetime64 or timedelta64) is
+/// UNSUPPORTED_DATA_TYPE, named by its Python type. NumPy is looked up in 
`sys.modules`, as a
+/// NumPy scalar exists only when NumPy is imported.
+fn numpy_scalar_literal(obj: &Bound<'_, PyAny>) -> 
PyResult<Option<LiteralExpression>> {
+    let py = obj.py();
+    let modules = py.import("sys")?.getattr("modules")?;
+    let numpy = match 
modules.cast::<pyo3::types::PyDict>()?.get_item("numpy")? {
+        Some(numpy) if !numpy.is_none() => numpy,
+        _ => return Ok(None),
+    };
+    if !obj.is_instance(&numpy.getattr("generic")?)? {
+        return Ok(None);
+    }
+    if obj.is_instance(&numpy.getattr("bool_")?)? {
+        return Ok(Some(LiteralExpression::Boolean(obj.is_truthy()?)));
+    }
+    let dtype = obj.getattr("dtype")?;
+    let kind: String = dtype.getattr("kind")?.extract()?;
+    let itemsize: usize = dtype.getattr("itemsize")?.extract()?;
+    let literal = match (kind.as_str(), itemsize) {
+        ("U", _) | ("S", _) => return Ok(None),
+        ("i", 1) => LiteralExpression::Byte(obj.extract::<i64>()? as i32),
+        ("i", 2) => LiteralExpression::Short(obj.extract::<i64>()? as i32),
+        ("i", 4) => LiteralExpression::Integer(obj.extract::<i64>()? as i32),
+        ("i", 8) => LiteralExpression::Long(obj.extract::<i64>()?),
+        ("f", 4) => LiteralExpression::Float(obj.extract::<f64>()? as f32),
+        ("f", 8) => LiteralExpression::Double(obj.extract::<f64>()?),
+        _ => {
+            let type_name = obj
+                .get_type()
+                .name()
+                .map(|n| n.to_string())
+                .unwrap_or_else(|_| "<unknown>".to_string());
+            return Err(crate::column::raise_pyspark(
+                py,
+                "PySparkTypeError",
+                "UNSUPPORTED_DATA_TYPE",
+                &[("data_type", &type_name)],
+            ));
+        }
+    };
+    Ok(Some(literal))
+}
+
 /// Build a literal for the non-primitive scalar Python types (`datetime`, 
`date`,
 /// `decimal.Decimal`, `bytes`/`bytearray`). Returns `None` if `obj` is not 
one of them,
 /// so the caller falls through to the primitive int/float/str/bool checks. 
Kept in sync
diff --git a/python/pyspark/sql/avro/functions.py 
b/python/pyspark/sql/avro/functions.py
index 300abe3..2a568b6 100644
--- a/python/pyspark/sql/avro/functions.py
+++ b/python/pyspark/sql/avro/functions.py
@@ -14,9 +14,69 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-"""Avro functions, mirroring ``pyspark.sql.avro.functions`` (the 
implementations live
-in ``pyspark.sql.functions``; re-exported here for the official import 
path)."""
+"""Avro functions, mirroring ``pyspark.sql.avro.functions``."""
+
+from pyspark import _pyspark
+from pyspark.errors import PySparkTypeError
+from pyspark.sql.column import Column
+from pyspark.sql.functions import _call_function, _dict_to_options_column, 
_to_col
+
+_functions = _pyspark.functions
+_pyfunc_from_avro = _functions.pyfunc_from_avro
+_pyfunc_from_avro_with_options = _functions.pyfunc_from_avro_with_options
+_pyfunc_to_avro_with_schema = _functions.pyfunc_to_avro_with_schema
+
+
+def _check_data(data):
+    if not isinstance(data, (Column, str)):
+        raise PySparkTypeError(
+            errorClass="INVALID_TYPE",
+            messageParameters={"arg_name": "data", "arg_type": 
"pyspark.sql.Column or str"},
+        )
+
+
+def from_avro(data, jsonFormatSchema, options=None):
+    """Converts a binary column of Avro format into its corresponding catalyst 
value.
+
+    Args:
+        data: the binary column (a Column or a column name).
+        jsonFormatSchema: the Avro schema in JSON string format.
+        options: optional dict of options for the Avro reader.
+    """
+    _check_data(data)
+    if not isinstance(jsonFormatSchema, str):
+        raise PySparkTypeError(
+            errorClass="INVALID_TYPE",
+            messageParameters={"arg_name": "jsonFormatSchema", "arg_type": 
"str"},
+        )
+    if options is not None and not isinstance(options, dict):
+        raise PySparkTypeError(
+            errorClass="INVALID_TYPE",
+            messageParameters={"arg_name": "options", "arg_type": "dict, 
optional"},
+        )
+    if options is None:
+        return _pyfunc_from_avro(_to_col(data), jsonFormatSchema)
+    return _pyfunc_from_avro_with_options(
+        _to_col(data), jsonFormatSchema, _dict_to_options_column(options)
+    )
+
+
+def to_avro(data, jsonFormatSchema=""):
+    """Converts a column into binary of Avro format.
+
+    Args:
+        data: the data column (a Column or a column name).
+        jsonFormatSchema: the Avro schema in JSON string format; empty to 
infer it.
+    """
+    _check_data(data)
+    if not isinstance(jsonFormatSchema, str):
+        raise PySparkTypeError(
+            errorClass="INVALID_TYPE",
+            messageParameters={"arg_name": "jsonFormatSchema", "arg_type": 
"str"},
+        )
+    if jsonFormatSchema == "":
+        return _call_function("to_avro", _to_col(data))
+    return _pyfunc_to_avro_with_schema(_to_col(data), jsonFormatSchema)
 
-from pyspark.sql.functions import from_avro, to_avro
 
 __all__ = ["from_avro", "to_avro"]
diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py
index f362437..7e2034d 100644
--- a/python/pyspark/sql/functions.py
+++ b/python/pyspark/sql/functions.py
@@ -1,8 +1,40 @@
 """pyspark.sql.functions module exposing Rust-backed SQL functions."""
 
-import os
+import decimal
+import functools
+import importlib.util as _importlib_util
+import inspect
+import os as _os
+import sys
+import warnings
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Iterable,
+    Mapping,
+    Optional,
+    Sequence,
+    Tuple,
+    Type,
+    Union,
+    ValuesView,
+    cast,
+    overload,
+)
 
 from pyspark import _pyspark
+from pyspark.errors import PySparkTypeError, PySparkValueError
+from pyspark.sql.column import Column
+from pyspark.sql.types import (
+    ArrayType,
+    ByteType,
+    DataType,
+    MapType,
+    NumericType,
+    StringType,
+    StructType,
+)
 
 _functions = getattr(_pyspark, 'functions', None)
 
@@ -16,18 +48,6 @@ _pyfunc_expr = _functions.pyfunc_expr
 _pyfunc_count = _functions.pyfunc_count
 _pyfunc_when = _functions.pyfunc_when
 _call_function = _functions.pyfunc_call_function
-# Mixed function bindings
-_pyfunc_from_avro = _functions.pyfunc_from_avro
-_pyfunc_from_avro_with_options = _functions.pyfunc_from_avro_with_options
-_pyfunc_to_avro_with_schema = _functions.pyfunc_to_avro_with_schema
-_pyfunc_from_protobuf = _functions.pyfunc_from_protobuf
-_pyfunc_from_protobuf_with_descriptor = 
_functions.pyfunc_from_protobuf_with_descriptor
-_pyfunc_from_protobuf_with_descriptor_and_options = 
_functions.pyfunc_from_protobuf_with_descriptor_and_options
-_pyfunc_from_protobuf_with_options = 
_functions.pyfunc_from_protobuf_with_options
-_pyfunc_to_protobuf = _functions.pyfunc_to_protobuf
-_pyfunc_to_protobuf_with_descriptor = 
_functions.pyfunc_to_protobuf_with_descriptor
-_pyfunc_to_protobuf_with_descriptor_and_options = 
_functions.pyfunc_to_protobuf_with_descriptor_and_options
-_pyfunc_to_protobuf_with_options = _functions.pyfunc_to_protobuf_with_options
 
 # Utility functions for wrapping/unwrapping columns
 def _unwrap(obj):
@@ -290,8 +310,8 @@ def _dict_to_options_column(options_dict):
         # Convert dict to alternating keys and values: [k1, v1, k2, v2, ...]
         args = []
         for k, v in options_dict.items():
-            args.append(k)
-            args.append(v)
+            args.append(str(k))
+            args.append(str(v))
         # Use create_map with literal strings - we'll build the Column through 
_call_function
         # since create_map is auto-generated
         return _call_function("create_map", *[_pyfunc_lit(arg) for arg in 
args])
@@ -380,94 +400,12 @@ def when(condition, value):
     return _pyfunc_when(condition, value if isinstance(value, _pyspark.Column) 
else lit(value))
 
 
-# Mixed/special functions - explicit wrappers for non-generic dispatch
-def from_avro(data, json_format_schema):
-    """Deserialize Avro data into a column.
-
-    Args:
-        data: column containing binary Avro data
-        json_format_schema: JSON string schema for the Avro data
-    """
-    return _pyfunc_from_avro(_to_col(data), json_format_schema)
-
-def from_avro_with_options(data, json_format_schema, options=None):
-    """Deserialize Avro data into a column with options.
-
-    Args:
-        data: column containing binary Avro data
-        json_format_schema: JSON string schema for the Avro data
-        options: dict or Column of options
-    """
-    if options is None:
-        return _pyfunc_from_avro(_to_col(data), json_format_schema)
-    options_col = _dict_to_options_column(options)
-    return _pyfunc_from_avro_with_options(_to_col(data), json_format_schema, 
options_col)
-
-def to_avro_with_schema(data, json_format_schema):
-    """Serialize a column to Avro binary format with schema.
-
-    Args:
-        data: column to serialize
-        json_format_schema: JSON string schema for the Avro data
-    """
-    return _pyfunc_to_avro_with_schema(_to_col(data), json_format_schema)
-
-def from_protobuf(data, message_name, desc_file_path=None, options=None, 
binary_descriptor_set=None):
-    """Deserialize Protobuf data into a column.
-
-    Args:
-        data: column containing binary Protobuf data
-        message_name: name of the Protobuf message type
-        desc_file_path: optional, path to descriptor file
-        options: optional, dict or Column of options
-        binary_descriptor_set: optional, binary descriptor set bytes
-    """
-    if binary_descriptor_set is not None and options is not None:
-        options_col = _dict_to_options_column(options)
-        return _pyfunc_from_protobuf_with_descriptor_and_options(
-            _to_col(data), message_name, binary_descriptor_set, options_col
-        )
-    elif binary_descriptor_set is not None:
-        return _pyfunc_from_protobuf_with_descriptor(
-            _to_col(data), message_name, binary_descriptor_set
-        )
-    elif options is not None:
-        options_col = _dict_to_options_column(options)
-        return _pyfunc_from_protobuf_with_options(_to_col(data), message_name, 
options_col)
-    else:
-        return _pyfunc_from_protobuf(_to_col(data), message_name)
-
-def to_protobuf(data, message_name, desc_file_path=None, options=None, 
binary_descriptor_set=None):
-    """Serialize a column to Protobuf binary format.
-
-    Args:
-        data: column to serialize
-        message_name: name of the Protobuf message type
-        desc_file_path: optional, path to descriptor file
-        options: optional, dict or Column of options
-        binary_descriptor_set: optional, binary descriptor set bytes
-    """
-    if binary_descriptor_set is not None and options is not None:
-        options_col = _dict_to_options_column(options)
-        return _pyfunc_to_protobuf_with_descriptor_and_options(
-            _to_col(data), message_name, binary_descriptor_set, options_col
-        )
-    elif binary_descriptor_set is not None:
-        return _pyfunc_to_protobuf_with_descriptor(
-            _to_col(data), message_name, binary_descriptor_set
-        )
-    elif options is not None:
-        options_col = _dict_to_options_column(options)
-        return _pyfunc_to_protobuf_with_options(_to_col(data), message_name, 
options_col)
-    else:
-        return _pyfunc_to_protobuf(_to_col(data), message_name)
-
 # Now we can import generated wrappers (they use _create_wrapper defined above)
 # We do this via exec to make _create_wrapper available in the generated 
module's namespace
-import importlib.util
-spec = importlib.util.spec_from_file_location("functions_generated",
-                                               
os.path.join(os.path.dirname(__file__), "functions_generated.py"))
-_gen_module = importlib.util.module_from_spec(spec)
+_spec = _importlib_util.spec_from_file_location(
+    "functions_generated", _os.path.join(_os.path.dirname(__file__), 
"functions_generated.py")
+)
+_gen_module = _importlib_util.module_from_spec(_spec)
 # Inject our helper functions into the generated module's namespace before 
loading. The
 # functions generated from their PySpark Connect implementations call lit, 
column and
 # _invoke_function (an UnresolvedFunction of the name and arguments), as 
PySpark Connect's
@@ -481,11 +419,14 @@ _gen_module._call_function = _call_function
 _gen_module.lit = lit
 _gen_module.column = _pyfunc_col
 _gen_module._invoke_function = _functions.pyfunc_invoke_function
-spec.loader.exec_module(_gen_module)
+_spec.loader.exec_module(_gen_module)
 
-# Import all generated functions into this module
+# Import all generated functions into this module, except those that PySpark 
does not expose
+# here: to_avro is pyspark.sql.avro.functions.to_avro, which takes the schema 
of its data.
+_NOT_IN_PYSPARK_FUNCTIONS = ("to_avro",)
 for name in _gen_module.__all__:
-    globals()[name] = getattr(_gen_module, name)
+    if name not in _NOT_IN_PYSPARK_FUNCTIONS:
+        globals()[name] = getattr(_gen_module, name)
 
 # PySpark Connect's _to_col, which raises PySparkTypeError NOT_EXPECTED_TYPE 
for an
 # argument that is neither a Column nor a column name.
@@ -808,34 +749,576 @@ except Exception:  # pragma: no cover
     pass
 from pyspark.sql.pandas.functions import PandasUDFType, ArrowUDFType  # noqa: 
E402,F401
 
-# Build __all__ with all function names
+# The public names of PySpark 4.3.0's pyspark.sql.functions, in its order. A 
star import
+# brings in exactly these; the other names of this module (e.g. chr, random, 
uuid, which
+# PySpark also has as attributes) are not exported, so they do not shadow the 
builtins and
+# the stdlib modules of the same names.
 __all__ = [
+    "broadcast",
+    "call_function",
     "col",
+    "column",
     "lit",
     "expr",
-    "sum",
-    "count",
+    "coalesce",
+    "ifnull",
+    "nanvl",
+    "nullif",
+    "nullifzero",
+    "nvl",
+    "nvl2",
+    "when",
+    "zeroifnull",
+    "equal_null",
+    "ilike",
+    "isnan",
+    "isnotnull",
+    "isnull",
+    "like",
+    "regexp",
+    "regexp_like",
+    "rlike",
+    "asc",
+    "asc_nulls_first",
+    "asc_nulls_last",
+    "desc",
+    "desc_nulls_first",
+    "desc_nulls_last",
+    "abs",
+    "acos",
+    "acosh",
+    "asin",
+    "asinh",
+    "atan",
+    "atan2",
+    "atanh",
+    "bin",
+    "bround",
+    "cbrt",
+    "ceil",
+    "ceiling",
+    "conv",
+    "cos",
+    "cosh",
+    "cot",
+    "csc",
+    "degrees",
+    "e",
+    "exp",
+    "expm1",
+    "factorial",
+    "floor",
+    "greatest",
+    "hex",
+    "hypot",
+    "least",
+    "ln",
+    "log",
+    "log10",
+    "log1p",
+    "log2",
+    "negate",
+    "negative",
+    "pi",
+    "pmod",
+    "positive",
+    "pow",
+    "power",
+    "radians",
+    "rand",
+    "randn",
+    "rint",
+    "round",
+    "sec",
+    "sign",
+    "signum",
+    "sin",
+    "sinh",
+    "sqrt",
+    "tan",
+    "tanh",
+    "try_add",
+    "try_divide",
+    "try_mod",
+    "try_multiply",
+    "try_subtract",
+    "unhex",
+    "uniform",
+    "width_bucket",
+    "ascii",
+    "base64",
+    "bit_length",
+    "btrim",
+    "char",
+    "char_length",
+    "character_length",
+    "collate",
+    "collation",
+    "concat_ws",
+    "contains",
+    "decode",
+    "elt",
+    "encode",
+    "endswith",
+    "find_in_set",
+    "format_number",
+    "format_string",
+    "from_base32",
+    "initcap",
+    "instr",
+    "is_valid_utf8",
+    "jaro_winkler_similarity",
+    "lcase",
+    "left",
+    "length",
+    "levenshtein",
+    "locate",
+    "lower",
+    "lpad",
+    "ltrim",
+    "make_valid_utf8",
+    "mask",
+    "octet_length",
+    "overlay",
+    "position",
+    "printf",
+    "quote",
+    "randstr",
+    "regexp_count",
+    "regexp_extract",
+    "regexp_extract_all",
+    "regexp_instr",
+    "regexp_replace",
+    "regexp_substr",
+    "repeat",
+    "replace",
+    "right",
+    "rpad",
+    "rtrim",
+    "sentences",
+    "soundex",
+    "split",
+    "split_part",
+    "startswith",
+    "substr",
+    "substring",
+    "substring_index",
+    "to_base32",
+    "to_binary",
+    "to_char",
+    "to_number",
+    "to_varchar",
+    "translate",
+    "trim",
+    "try_to_binary",
+    "try_to_number",
+    "try_validate_utf8",
+    "ucase",
+    "unbase64",
+    "upper",
+    "validate_utf8",
+    "bit_count",
+    "bit_get",
+    "bitwise_not",
+    "getbit",
+    "shiftleft",
+    "shiftright",
+    "shiftrightunsigned",
+    "add_months",
+    "convert_timezone",
+    "curdate",
+    "current_date",
+    "current_time",
+    "current_timestamp",
+    "current_timezone",
+    "date_add",
+    "date_diff",
+    "date_format",
+    "date_from_unix_date",
+    "date_part",
+    "date_sub",
+    "date_trunc",
+    "dateadd",
+    "datediff",
+    "datepart",
+    "day",
+    "dayname",
+    "dayofmonth",
+    "dayofweek",
+    "dayofyear",
+    "extract",
+    "from_unixtime",
+    "from_utc_timestamp",
+    "hour",
+    "last_day",
+    "localtimestamp",
+    "make_date",
+    "make_dt_interval",
+    "make_interval",
+    "make_time",
+    "make_timestamp",
+    "make_timestamp_ltz",
+    "make_timestamp_ntz",
+    "make_ym_interval",
+    "minute",
+    "month",
+    "monthname",
+    "months_between",
+    "next_day",
+    "now",
+    "quarter",
+    "second",
+    "session_window",
+    "timestamp_add",
+    "timestamp_diff",
+    "timestamp_micros",
+    "timestamp_millis",
+    "timestamp_nanos",
+    "timestamp_seconds",
+    "time_bucket",
+    "time_diff",
+    "time_from_micros",
+    "time_from_millis",
+    "time_from_seconds",
+    "time_to_micros",
+    "time_to_millis",
+    "time_to_seconds",
+    "time_trunc",
+    "to_date",
+    "to_time",
+    "to_timestamp",
+    "to_timestamp_ltz",
+    "to_timestamp_ntz",
+    "to_unix_timestamp",
+    "to_utc_timestamp",
+    "trunc",
+    "try_make_interval",
+    "try_make_timestamp",
+    "try_make_timestamp_ltz",
+    "try_make_timestamp_ntz",
+    "try_to_time",
+    "try_to_timestamp",
+    "try_to_date",
+    "unix_date",
+    "unix_micros",
+    "unix_millis",
+    "unix_nanos",
+    "unix_seconds",
+    "unix_timestamp",
+    "weekday",
+    "weekofyear",
+    "window",
+    "window_time",
+    "year",
+    "crc32",
+    "hash",
+    "md5",
+    "sha",
+    "sha1",
+    "sha2",
+    "xxhash64",
+    "aggregate",
+    "array_sort",
+    "cardinality",
+    "concat",
+    "element_at",
+    "exists",
+    "filter",
+    "forall",
+    "map_filter",
+    "map_zip_with",
+    "reduce",
+    "reverse",
+    "size",
+    "transform",
+    "transform_keys",
+    "transform_values",
+    "try_element_at",
+    "zip_with",
+    "array",
+    "array_append",
+    "array_compact",
+    "array_contains",
+    "array_distinct",
+    "array_except",
+    "array_insert",
+    "array_intersect",
+    "array_join",
+    "array_max",
+    "array_min",
+    "array_position",
+    "array_prepend",
+    "array_remove",
+    "array_repeat",
+    "array_size",
+    "array_union",
+    "arrays_overlap",
+    "arrays_zip",
+    "flatten",
+    "get",
+    "sequence",
+    "shuffle",
+    "slice",
+    "sort_array",
+    "named_struct",
+    "struct",
+    "create_map",
+    "map_concat",
+    "map_contains_key",
+    "map_entries",
+    "map_from_arrays",
+    "map_from_entries",
+    "map_keys",
+    "map_values",
+    "str_to_map",
+    "any_value",
+    "approx_count_distinct",
+    "approx_percentile",
+    "array_agg",
     "avg",
+    "bit_and",
+    "bit_or",
+    "bit_xor",
+    "bitmap_and_agg",
+    "bitmap_construct_agg",
+    "bitmap_or_agg",
+    "bool_and",
+    "bool_or",
+    "collect_list",
+    "collect_set",
+    "collect_union",
+    "corr",
+    "count",
+    "count_distinct",
+    "count_if",
+    "count_min_sketch",
+    "covar_pop",
+    "covar_samp",
+    "every",
+    "first",
+    "first_value",
+    "grouping",
+    "grouping_id",
+    "histogram_numeric",
+    "hll_sketch_agg",
+    "hll_union_agg",
+    "kll_sketch_agg_bigint",
+    "kll_sketch_agg_double",
+    "kll_sketch_agg_float",
+    "kll_merge_agg_bigint",
+    "kll_merge_agg_double",
+    "kll_merge_agg_float",
+    "kurtosis",
+    "last",
+    "last_value",
+    "listagg",
+    "listagg_distinct",
     "max",
+    "max_by",
+    "mean",
+    "median",
     "min",
-    "when",
-    "udf",
+    "min_by",
+    "mode",
+    "percentile",
+    "percentile_approx",
+    "product",
+    "regr_avgx",
+    "regr_avgy",
+    "regr_count",
+    "regr_intercept",
+    "regr_r2",
+    "regr_slope",
+    "regr_sxx",
+    "regr_sxy",
+    "regr_syy",
+    "skewness",
+    "some",
+    "std",
+    "stddev",
+    "stddev_pop",
+    "stddev_samp",
+    "string_agg",
+    "string_agg_distinct",
+    "sum",
+    "sum_distinct",
+    "theta_intersection_agg",
+    "theta_sketch_agg",
+    "theta_union_agg",
+    "tuple_intersection_agg_double",
+    "tuple_intersection_agg_integer",
+    "tuple_sketch_agg_double",
+    "tuple_sketch_agg_integer",
+    "tuple_union_agg_double",
+    "tuple_union_agg_integer",
+    "try_avg",
+    "try_sum",
+    "var_pop",
+    "var_samp",
+    "variance",
+    "counter_diff",
+    "cume_dist",
+    "dense_rank",
+    "lag",
+    "lead",
+    "nth_value",
+    "ntile",
+    "percent_rank",
+    "rank",
+    "row_number",
+    "explode",
+    "explode_outer",
+    "inline",
+    "inline_outer",
+    "posexplode",
+    "posexplode_outer",
+    "stack",
+    "years",
+    "months",
+    "days",
+    "hours",
+    "bucket",
+    "from_csv",
+    "schema_of_csv",
+    "to_csv",
+    "from_json",
+    "get_json_object",
+    "json_array_length",
+    "json_object_keys",
+    "json_tuple",
+    "schema_of_json",
+    "to_json",
+    "is_valid_variant",
+    "is_variant_null",
+    "parse_json",
+    "schema_of_variant",
+    "schema_of_variant_agg",
+    "try_variant_get",
+    "variant_array_append",
+    "try_variant_array_append",
+    "variant_delete",
+    "variant_strip_nulls",
+    "variant_get",
+    "variant_insert",
+    "try_variant_insert",
+    "variant_set",
+    "try_variant_set",
+    "try_parse_json",
+    "to_variant_object",
+    "from_xml",
+    "schema_of_xml",
+    "to_xml",
+    "xpath",
+    "xpath_boolean",
+    "xpath_double",
+    "xpath_float",
+    "xpath_int",
+    "xpath_long",
+    "xpath_number",
+    "xpath_short",
+    "xpath_string",
+    "parse_url",
+    "try_parse_url",
+    "url_decode",
+    "url_encode",
+    "try_url_decode",
+    "aes_decrypt",
+    "aes_encrypt",
+    "assert_true",
+    "bitmap_bit_position",
+    "bitmap_bucket_number",
+    "bitmap_count",
+    "current_catalog",
+    "current_database",
+    "current_path",
+    "current_schema",
+    "current_user",
+    "hmac",
+    "input_file_block_length",
+    "input_file_block_start",
+    "input_file_name",
+    "java_method",
+    "monotonically_increasing_id",
+    "raise_error",
+    "reflect",
+    "session_user",
+    "spark_partition_id",
+    "try_aes_decrypt",
+    "try_reflect",
+    "typeof",
+    "user",
+    "version",
+    "AnalyzeArgument",
+    "AnalyzeResult",
+    "ArrowUDFType",
+    "OrderingColumn",
+    "PandasUDFType",
+    "PartitioningColumn",
+    "SelectedColumn",
+    "SkipRestOfInputTableException",
+    "UserDefinedFunction",
+    "UserDefinedTableFunction",
+    "arrow_udf",
+    "hll_sketch_estimate",
+    "hll_union",
+    "kll_sketch_get_n_bigint",
+    "kll_sketch_get_n_double",
+    "kll_sketch_get_n_float",
+    "kll_sketch_get_quantile_bigint",
+    "kll_sketch_get_quantile_double",
+    "kll_sketch_get_quantile_float",
+    "kll_sketch_get_rank_bigint",
+    "kll_sketch_get_rank_double",
+    "kll_sketch_get_rank_float",
+    "kll_sketch_merge_bigint",
+    "kll_sketch_merge_double",
+    "kll_sketch_merge_float",
+    "kll_sketch_to_string_bigint",
+    "kll_sketch_to_string_double",
+    "kll_sketch_to_string_float",
+    "theta_difference",
+    "theta_intersection",
+    "theta_sketch_estimate",
+    "theta_union",
+    "tuple_difference_double",
+    "tuple_difference_integer",
+    "tuple_difference_theta_double",
+    "tuple_difference_theta_integer",
+    "tuple_intersection_double",
+    "tuple_intersection_integer",
+    "tuple_intersection_theta_double",
+    "tuple_intersection_theta_integer",
+    "tuple_sketch_estimate_double",
+    "tuple_sketch_estimate_integer",
+    "tuple_sketch_summary_double",
+    "tuple_sketch_summary_integer",
+    "tuple_sketch_theta_double",
+    "tuple_sketch_theta_integer",
+    "tuple_union_double",
+    "tuple_union_integer",
+    "tuple_union_theta_double",
+    "tuple_union_theta_integer",
+    "st_asbinary",
+    "st_geogfromwkb",
+    "st_geomfromwkb",
+    "st_setsrid",
+    "st_srid",
+    "vector_cosine_similarity",
+    "vector_inner_product",
+    "vector_l2_distance",
+    "vector_norm",
+    "vector_normalize",
+    "vector_avg",
+    "vector_sum",
+    "call_udf",
     "pandas_udf",
+    "udf",
+    "udtf",
+    "arrow_udtf",
+    "unwrap_udt",
 ]
-
-# Add all other function names (dynamically)
-import inspect
-_export_globals = list(globals().items())
-_exported_names = set(__all__)
-for name, obj in _export_globals:
-    if (callable(obj) and
-        not name.startswith('_') and
-        name not in _exported_names and
-        name not in ['inspect', 'importlib', '_gen_module', 'spec', 'os']):
-        __all__.append(name)
-        _exported_names.add(name)
 # Leave no loop variables behind, as PySpark has no `functions.name` or 
`functions.obj`.
-del _export_globals, _exported_names, name, obj
+del name
 
 # Expose the `builtin` submodule at ``pyspark.sql.functions.builtin``. 
Upstream v4.2.0 splits
 # functions into a package whose ``builtin`` submodule holds the actual 
functions and whose
diff --git a/python/pyspark/sql/protobuf/functions.py 
b/python/pyspark/sql/protobuf/functions.py
index dfd4f85..a389602 100644
--- a/python/pyspark/sql/protobuf/functions.py
+++ b/python/pyspark/sql/protobuf/functions.py
@@ -14,9 +14,88 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-"""Protobuf functions, mirroring ``pyspark.sql.protobuf.functions`` 
(implementations
-live in ``pyspark.sql.functions``; re-exported here for the official import 
path)."""
+"""Protobuf functions, mirroring ``pyspark.sql.protobuf.functions``."""
+
+from pyspark import _pyspark
+from pyspark.sql.functions import _dict_to_options_column, _to_col
+
+_functions = _pyspark.functions
+_pyfunc_from_protobuf = _functions.pyfunc_from_protobuf
+_pyfunc_from_protobuf_with_descriptor = 
_functions.pyfunc_from_protobuf_with_descriptor
+_pyfunc_from_protobuf_with_descriptor_and_options = (
+    _functions.pyfunc_from_protobuf_with_descriptor_and_options
+)
+_pyfunc_from_protobuf_with_options = 
_functions.pyfunc_from_protobuf_with_options
+_pyfunc_to_protobuf = _functions.pyfunc_to_protobuf
+_pyfunc_to_protobuf_with_descriptor = 
_functions.pyfunc_to_protobuf_with_descriptor
+_pyfunc_to_protobuf_with_descriptor_and_options = (
+    _functions.pyfunc_to_protobuf_with_descriptor_and_options
+)
+_pyfunc_to_protobuf_with_options = _functions.pyfunc_to_protobuf_with_options
+
+
+def _descriptor_set(descFilePath, binaryDescriptorSet):
+    """The binary descriptor set: the given bytes, else the content of the 
descriptor file."""
+    if binaryDescriptorSet is not None:
+        return binaryDescriptorSet
+    if descFilePath is not None:
+        with open(descFilePath, "rb") as f:
+            return f.read()
+    return None
+
+
+def from_protobuf(
+    data, messageName, descFilePath=None, options=None, 
binaryDescriptorSet=None
+):
+    """Converts a binary column of Protobuf format into its corresponding 
catalyst value.
+
+    Args:
+        data: the binary column (a Column or a column name).
+        messageName: the protobuf message name to look for in the descriptor 
file.
+        descFilePath: the protobuf descriptor file path.
+        options: optional dict of options for the Protobuf reader.
+        binaryDescriptorSet: the protobuf descriptor set as bytes, instead of 
a file.
+    """
+    descriptor = _descriptor_set(descFilePath, binaryDescriptorSet)
+    column = _to_col(data)
+    if descriptor is None:
+        if options is None:
+            return _pyfunc_from_protobuf(column, messageName)
+        return _pyfunc_from_protobuf_with_options(
+            column, messageName, _dict_to_options_column(options)
+        )
+    if options is None:
+        return _pyfunc_from_protobuf_with_descriptor(column, messageName, 
descriptor)
+    return _pyfunc_from_protobuf_with_descriptor_and_options(
+        column, messageName, descriptor, _dict_to_options_column(options)
+    )
+
+
+def to_protobuf(
+    data, messageName, descFilePath=None, options=None, 
binaryDescriptorSet=None
+):
+    """Converts a column into binary of Protobuf format.
+
+    Args:
+        data: the data column (a Column or a column name).
+        messageName: the protobuf message name to look for in the descriptor 
file.
+        descFilePath: the protobuf descriptor file path.
+        options: optional dict of options for the Protobuf writer.
+        binaryDescriptorSet: the protobuf descriptor set as bytes, instead of 
a file.
+    """
+    descriptor = _descriptor_set(descFilePath, binaryDescriptorSet)
+    column = _to_col(data)
+    if descriptor is None:
+        if options is None:
+            return _pyfunc_to_protobuf(column, messageName)
+        return _pyfunc_to_protobuf_with_options(
+            column, messageName, _dict_to_options_column(options)
+        )
+    if options is None:
+        return _pyfunc_to_protobuf_with_descriptor(column, messageName, 
descriptor)
+    return _pyfunc_to_protobuf_with_descriptor_and_options(
+        column, messageName, descriptor, _dict_to_options_column(options)
+    )
 
-from pyspark.sql.functions import from_protobuf, to_protobuf
 
 __all__ = ["from_protobuf", "to_protobuf"]
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index deb3ef3..a2efb5e 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -104,6 +104,15 @@ def test_conf_get_with_a_none_default(session):
         session.conf.get(key)
 
 
+def 
test_numpy_scalar_literals_have_the_types_of_their_dtypes_as_pyspark(session):
+    np = pytest.importorskip("numpy")
+    from pyspark.sql import functions as F
+
+    # As PySpark: a NumPy scalar's literal is typed by its dtype, as the 
server reports it.
+    df = session.range(1).select(F.lit(np.int64(5)), F.lit(np.float32(1.5)), 
F.lit(np.int8(5)))
+    assert [f.dataType.simpleString() for f in df.schema.fields] == ["bigint", 
"float", "tinyint"]
+
+
 def test_user_id_of_the_connection_string_reaches_every_request_as_pyspark():
     # The server keys a session by the user ID in each request. As in PySpark, 
the
     # queries carry the user ID of the connection string, as the configs do, 
so a config
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 7e531bd..9d4bd24 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -560,12 +560,11 @@ def test_functions_exports_fresh_process():
         from pyspark.sql import functions as F
         assert not hasattr(F, "name") and not hasattr(F, "obj")
 
-        initial = ["col", "lit", "expr", "sum", "count", "avg", "max", "min",
-                   "when", "udf", "pandas_udf"]
+        official = __OFFICIAL_FUNCTIONS_ALL__
 
         def check_exports():
             assert type(F.__all__) is list
-            assert F.__all__[:len(initial)] == initial
+            assert F.__all__ == official
             assert len(F.__all__) == len(set(F.__all__))
             assert all(callable(getattr(F, name)) for name in F.__all__)
             assert not any(name.startswith("_") for name in F.__all__)
@@ -605,7 +604,7 @@ def test_functions_exports_fresh_process():
         importlib.reload(F)
         assert F.__all__ is not old_all
         assert check_exports() == expected
-    ''')], capture_output=True, text=True)
+    ''').replace("__OFFICIAL_FUNCTIONS_ALL__", repr(OFFICIAL_FUNCTIONS_ALL))], 
capture_output=True, text=True)
     assert result.returncode == 0, result.stdout + result.stderr
 
 
@@ -1294,11 +1293,14 @@ def test_functions_module_wrappers():
     assert type(F.sha2(c, 256)).__name__ == "Column"
     assert type(F.window(F.col("t"), "10 minutes")).__name__ == "Column"
     assert type(F.window(F.col("t"), "10 minutes", "5 minutes", "0 
seconds")).__name__ == "Column"
-    assert type(F.from_avro(c, "{}")).__name__ == "Column"
-    assert type(F.from_avro_with_options(c, "{}", {"mode": 
"PERMISSIVE"})).__name__ == "Column"
-    assert type(F.to_avro_with_schema(c, "{}")).__name__ == "Column"
-    assert type(F.from_protobuf(c, "M", options={"x": "y"})).__name__ == 
"Column"
-    assert type(F.to_protobuf(c, "M", options={"x": "y"})).__name__ == "Column"
+    import pyspark.sql.avro.functions as avro
+    import pyspark.sql.protobuf.functions as protobuf
+
+    assert type(avro.from_avro(c, "{}")).__name__ == "Column"
+    assert type(avro.from_avro(c, "{}", {"mode": "PERMISSIVE"})).__name__ == 
"Column"
+    assert type(avro.to_avro(c, "{}")).__name__ == "Column"
+    assert type(protobuf.from_protobuf(c, "M", options={"x": "y"})).__name__ 
== "Column"
+    assert type(protobuf.to_protobuf(c, "M", options={"x": "y"})).__name__ == 
"Column"
 
 
 def test_type_class_hierarchy():
@@ -2223,14 +2225,20 @@ def test_sql_is_remote():
     assert S.is_remote() is True
 
 
-def test_functions_protobuf_descriptor_branches():
+def test_functions_protobuf_descriptor_branches(tmp_path):
+    import pyspark.sql.protobuf.functions as protobuf
     from pyspark.sql import functions as F
+
     c = F.col("v")
-    # descriptor-file and binary-descriptor-set branches
-    assert type(F.from_protobuf(c, "M", 
desc_file_path="/tmp/x.desc")).__name__ == "Column"
-    assert type(F.from_protobuf(c, "M", 
binary_descriptor_set=b"\x00")).__name__ == "Column"
-    assert type(F.to_protobuf(c, "M", desc_file_path="/tmp/x.desc")).__name__ 
== "Column"
-    assert type(F.to_protobuf(c, "M", binary_descriptor_set=b"\x00")).__name__ 
== "Column"
+    descriptor = tmp_path / "m.desc"
+    descriptor.write_bytes(b"\x00")
+    # descriptor-file and binary-descriptor-set branches, with and without 
options
+    assert type(protobuf.from_protobuf(c, "M", 
descFilePath=str(descriptor))).__name__ == "Column"
+    assert type(protobuf.from_protobuf(c, "M", 
binaryDescriptorSet=b"\x00")).__name__ == "Column"
+    assert type(protobuf.to_protobuf(c, "M", 
descFilePath=str(descriptor))).__name__ == "Column"
+    assert type(protobuf.to_protobuf(c, "M", 
binaryDescriptorSet=b"\x00")).__name__ == "Column"
+    assert type(protobuf.from_protobuf(c, "M", descFilePath=str(descriptor), 
options={"x": "y"})).__name__ == "Column"
+    assert type(protobuf.to_protobuf(c, "M", binaryDescriptorSet=b"\x00", 
options={"x": "y"})).__name__ == "Column"
 
 
 def test_udf_defaults_and_registration():
@@ -7180,6 +7188,207 @@ def 
test_completion_propagates_schema_errors_and_can_retry(completion_analyzed,
     assert service.calls == 3
 
 
+# --------------------------------------------------------------------------- 
functions exports and literals of NumPy scalars
+
+# The public names of PySpark 4.3.0's pyspark.sql.functions, in its order 
(__all__ is static there).
+OFFICIAL_FUNCTIONS_ALL = """
+    broadcast call_function col column lit expr coalesce ifnull nanvl nullif 
nullifzero nvl
+    nvl2 when zeroifnull equal_null ilike isnan isnotnull isnull like regexp 
regexp_like
+    rlike asc asc_nulls_first asc_nulls_last desc desc_nulls_first 
desc_nulls_last abs acos
+    acosh asin asinh atan atan2 atanh bin bround cbrt ceil ceiling conv cos 
cosh cot csc
+    degrees e exp expm1 factorial floor greatest hex hypot least ln log log10 
log1p log2
+    negate negative pi pmod positive pow power radians rand randn rint round 
sec sign signum
+    sin sinh sqrt tan tanh try_add try_divide try_mod try_multiply 
try_subtract unhex
+    uniform width_bucket ascii base64 bit_length btrim char char_length 
character_length
+    collate collation concat_ws contains decode elt encode endswith find_in_set
+    format_number format_string from_base32 initcap instr is_valid_utf8
+    jaro_winkler_similarity lcase left length levenshtein locate lower lpad 
ltrim
+    make_valid_utf8 mask octet_length overlay position printf quote randstr 
regexp_count
+    regexp_extract regexp_extract_all regexp_instr regexp_replace 
regexp_substr repeat
+    replace right rpad rtrim sentences soundex split split_part startswith 
substr substring
+    substring_index to_base32 to_binary to_char to_number to_varchar translate 
trim
+    try_to_binary try_to_number try_validate_utf8 ucase unbase64 upper 
validate_utf8
+    bit_count bit_get bitwise_not getbit shiftleft shiftright 
shiftrightunsigned add_months
+    convert_timezone curdate current_date current_time current_timestamp 
current_timezone
+    date_add date_diff date_format date_from_unix_date date_part date_sub 
date_trunc dateadd
+    datediff datepart day dayname dayofmonth dayofweek dayofyear extract 
from_unixtime
+    from_utc_timestamp hour last_day localtimestamp make_date make_dt_interval 
make_interval
+    make_time make_timestamp make_timestamp_ltz make_timestamp_ntz 
make_ym_interval minute
+    month monthname months_between next_day now quarter second session_window 
timestamp_add
+    timestamp_diff timestamp_micros timestamp_millis timestamp_nanos 
timestamp_seconds
+    time_bucket time_diff time_from_micros time_from_millis time_from_seconds 
time_to_micros
+    time_to_millis time_to_seconds time_trunc to_date to_time to_timestamp 
to_timestamp_ltz
+    to_timestamp_ntz to_unix_timestamp to_utc_timestamp trunc try_make_interval
+    try_make_timestamp try_make_timestamp_ltz try_make_timestamp_ntz 
try_to_time
+    try_to_timestamp try_to_date unix_date unix_micros unix_millis unix_nanos 
unix_seconds
+    unix_timestamp weekday weekofyear window window_time year crc32 hash md5 
sha sha1 sha2
+    xxhash64 aggregate array_sort cardinality concat element_at exists filter 
forall
+    map_filter map_zip_with reduce reverse size transform transform_keys 
transform_values
+    try_element_at zip_with array array_append array_compact array_contains 
array_distinct
+    array_except array_insert array_intersect array_join array_max array_min 
array_position
+    array_prepend array_remove array_repeat array_size array_union 
arrays_overlap arrays_zip
+    flatten get sequence shuffle slice sort_array named_struct struct 
create_map map_concat
+    map_contains_key map_entries map_from_arrays map_from_entries map_keys 
map_values
+    str_to_map any_value approx_count_distinct approx_percentile array_agg avg 
bit_and
+    bit_or bit_xor bitmap_and_agg bitmap_construct_agg bitmap_or_agg bool_and 
bool_or
+    collect_list collect_set collect_union corr count count_distinct count_if
+    count_min_sketch covar_pop covar_samp every first first_value grouping 
grouping_id
+    histogram_numeric hll_sketch_agg hll_union_agg kll_sketch_agg_bigint
+    kll_sketch_agg_double kll_sketch_agg_float kll_merge_agg_bigint 
kll_merge_agg_double
+    kll_merge_agg_float kurtosis last last_value listagg listagg_distinct max 
max_by mean
+    median min min_by mode percentile percentile_approx product regr_avgx 
regr_avgy
+    regr_count regr_intercept regr_r2 regr_slope regr_sxx regr_sxy regr_syy 
skewness some
+    std stddev stddev_pop stddev_samp string_agg string_agg_distinct sum 
sum_distinct
+    theta_intersection_agg theta_sketch_agg theta_union_agg 
tuple_intersection_agg_double
+    tuple_intersection_agg_integer tuple_sketch_agg_double 
tuple_sketch_agg_integer
+    tuple_union_agg_double tuple_union_agg_integer try_avg try_sum var_pop 
var_samp variance
+    counter_diff cume_dist dense_rank lag lead nth_value ntile percent_rank 
rank row_number
+    explode explode_outer inline inline_outer posexplode posexplode_outer 
stack years months
+    days hours bucket from_csv schema_of_csv to_csv from_json get_json_object
+    json_array_length json_object_keys json_tuple schema_of_json to_json 
is_valid_variant
+    is_variant_null parse_json schema_of_variant schema_of_variant_agg 
try_variant_get
+    variant_array_append try_variant_array_append variant_delete 
variant_strip_nulls
+    variant_get variant_insert try_variant_insert variant_set try_variant_set 
try_parse_json
+    to_variant_object from_xml schema_of_xml to_xml xpath xpath_boolean 
xpath_double
+    xpath_float xpath_int xpath_long xpath_number xpath_short xpath_string 
parse_url
+    try_parse_url url_decode url_encode try_url_decode aes_decrypt aes_encrypt 
assert_true
+    bitmap_bit_position bitmap_bucket_number bitmap_count current_catalog 
current_database
+    current_path current_schema current_user hmac input_file_block_length
+    input_file_block_start input_file_name java_method 
monotonically_increasing_id
+    raise_error reflect session_user spark_partition_id try_aes_decrypt 
try_reflect typeof
+    user version AnalyzeArgument AnalyzeResult ArrowUDFType OrderingColumn 
PandasUDFType
+    PartitioningColumn SelectedColumn SkipRestOfInputTableException 
UserDefinedFunction
+    UserDefinedTableFunction arrow_udf hll_sketch_estimate hll_union 
kll_sketch_get_n_bigint
+    kll_sketch_get_n_double kll_sketch_get_n_float 
kll_sketch_get_quantile_bigint
+    kll_sketch_get_quantile_double kll_sketch_get_quantile_float 
kll_sketch_get_rank_bigint
+    kll_sketch_get_rank_double kll_sketch_get_rank_float 
kll_sketch_merge_bigint
+    kll_sketch_merge_double kll_sketch_merge_float kll_sketch_to_string_bigint
+    kll_sketch_to_string_double kll_sketch_to_string_float theta_difference
+    theta_intersection theta_sketch_estimate theta_union 
tuple_difference_double
+    tuple_difference_integer tuple_difference_theta_double 
tuple_difference_theta_integer
+    tuple_intersection_double tuple_intersection_integer 
tuple_intersection_theta_double
+    tuple_intersection_theta_integer tuple_sketch_estimate_double
+    tuple_sketch_estimate_integer tuple_sketch_summary_double 
tuple_sketch_summary_integer
+    tuple_sketch_theta_double tuple_sketch_theta_integer tuple_union_double
+    tuple_union_integer tuple_union_theta_double tuple_union_theta_integer 
st_asbinary
+    st_geogfromwkb st_geomfromwkb st_setsrid st_srid vector_cosine_similarity
+    vector_inner_product vector_l2_distance vector_norm vector_normalize 
vector_avg
+    vector_sum call_udf pandas_udf udf udtf arrow_udtf unwrap_udt
+""".split()
+
+
+def test_star_import_of_functions_exports_the_official_names_only():
+    # A star import brings in exactly the names of PySpark's __all__. The 
names that PySpark has
+    # only as attributes (chr, random, uuid, ...) are not exported, so they do 
not shadow the
+    # builtin chr and the stdlib random and uuid; nor are the avro and 
protobuf functions, which
+    # PySpark has in pyspark.sql.avro.functions and 
pyspark.sql.protobuf.functions.
+    import builtins
+
+    from pyspark.sql import functions as F
+
+    assert F.__all__ == OFFICIAL_FUNCTIONS_ALL
+    namespace = {}
+    exec("from pyspark.sql.functions import *", namespace)
+    namespace.pop("__builtins__")
+    assert list(namespace) == OFFICIAL_FUNCTIONS_ALL
+    for name in ("chr", "random", "uuid", "countDistinct", "bitwiseNOT", 
"toDegrees",
+                 "approxCountDistinct"):
+        assert name not in namespace
+        assert callable(getattr(F, name))
+    assert F.random is F.rand
+    assert "chr" not in namespace and builtins.chr(65) == "A"
+    for name in ("from_avro", "from_avro_with_options", "to_avro", 
"to_avro_with_schema",
+                 "from_protobuf", "to_protobuf"):
+        assert name not in namespace
+        assert not hasattr(F, name)
+
+
+def test_avro_and_protobuf_functions_are_on_their_official_modules(tmp_path, 
mock_connect):
+    import pyspark.sql.avro.functions as avro
+    import pyspark.sql.protobuf.functions as protobuf
+    from pyspark.errors import PySparkTypeError
+    from pyspark.sql import functions as F
+
+    spark, service = mock_connect
+    c = F.col("v")
+    assert type(avro.from_avro(c, "{}")).__name__ == "Column"
+    assert type(avro.from_avro("v", "{}", {"mode": "PERMISSIVE"})).__name__ == 
"Column"
+    assert type(avro.to_avro(c)).__name__ == "Column"
+    assert type(avro.to_avro(c, "{}")).__name__ == "Column"
+    with pytest.raises(PySparkTypeError) as error:
+        avro.from_avro(1, "{}")
+    assert error.value.getCondition() == "INVALID_TYPE"
+    assert error.value.getMessageParameters() == {
+        "arg_name": "data", "arg_type": "pyspark.sql.Column or str"}
+    # As PySpark's, the descriptor file is read (it was ignored before), and 
the bytes reach the
+    # server as the binary of the function's descriptor set.
+    descriptor = tmp_path / "m.desc"
+    descriptor.write_bytes(b"\x00fue-e-descriptor\xff")
+    spark.range(1).select(protobuf.from_protobuf(F.col("id"), "M", 
descFilePath=str(descriptor))).collect()
+    assert b"\x00fue-e-descriptor\xff" in service.execute_requests[-1]
+
+
+def 
test_numpy_scalar_literals_are_typed_by_their_dtype_as_pyspark(mock_connect):
+    import struct
+
+    np = pytest.importorskip("numpy")
+    from pyspark.sql import functions as F
+
+    spark, service = mock_connect
+    # As PySpark's LiteralExpression of a NumPy scalar, typed by its dtype: a 
bool, a signed int
+    # of 8 to 64 bits (tinyint to bigint), float32 (float) and float64 
(double). The Literal
+    # fields are: boolean=3, byte=4, short=5, integer=6, long=7, float=10, 
double=11, string=13,
+    # binary=2.
+    cases = [
+        (np.bool_(True), 3, 1),
+        (np.int8(5), 4, 5),
+        (np.int16(5), 5, 5),
+        (np.int32(5), 6, 5),
+        (np.int64(5), 7, 5),
+        (np.int64(2**40), 7, 2**40),
+        (np.float32(1.5), 10, struct.pack("<f", 1.5)),
+        (np.float64(1.5), 11, struct.pack("<d", 1.5)),
+        (np.str_("abc"), 13, b"abc"),
+        (np.bytes_(b"ab"), 2, b"ab"),
+        # A Python int and float keep their types: int (integer) and double.
+        (5, 6, 5),
+        (1.5, 11, struct.pack("<d", 1.5)),
+    ]
+    for value, field, expected in cases:
+        spark.range(1).select(F.lit(value)).collect()
+        plan = 
parse_message(parse_message(service.execute_requests[-1])[3][0])[1][0]
+        expression = 
parse_message(parse_message(parse_message(plan)[3][0])[3][0])
+        literal = parse_message(expression[1][0])
+        assert sorted(literal) == [field], repr(value)
+        assert literal[field] == [expected], repr(value)
+
+
+def test_numpy_scalars_of_other_types_are_unsupported_as_pyspark(mock_connect):
+    np = pytest.importorskip("numpy")
+    from pyspark.errors import PySparkTypeError
+    from pyspark.sql import functions as F
+
+    # As PySpark's, the unsupported NumPy scalars are UNSUPPORTED_DATA_TYPE, 
named by their
+    # Python type; and an array of an unsupported dtype is 
UNSUPPORTED_NUMPY_ARRAY_SCALAR.
+    for value, type_name in [
+        (np.uint8(5), "uint8"),
+        (np.uint64(5), "uint64"),
+        (np.float16(1.5), "float16"),
+        (np.longdouble(1.5), "longdouble"),
+        (np.complex128(1 + 2j), "complex128"),
+        (np.datetime64("2020-01-01"), "datetime64"),
+        (np.timedelta64(1, "D"), "timedelta64"),
+    ]:
+        with pytest.raises(PySparkTypeError) as error:
+            F.lit(value)
+        assert error.value.getCondition() == "UNSUPPORTED_DATA_TYPE"
+        assert error.value.getMessageParameters() == {"data_type": type_name}
+    with pytest.raises(PySparkTypeError) as error:
+        F.lit(np.array([1], dtype=np.uint8))
+    assert error.value.getCondition() == "UNSUPPORTED_NUMPY_ARRAY_SCALAR"
+    assert error.value.getMessageParameters() == {"dtype": "uint8"}
+
+
 # --------------------------------------------------------------------------- 
4.3.0 gaps
 
 


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to