This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 9d9b0e7 [SPARK-61065][CONNECT][PYTHON] Export the names of PySpark
from pyspark.sql.functions, and type the literals of NumPy scalars by their
dtype, as PySpark does
9d9b0e7 is described below
commit 9d9b0e7775f818a79ed2f88bf1d977b0bfddfbb9
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 11:42:24 2026 +0900
[SPARK-61065][CONNECT][PYTHON] Export the names of PySpark from
pyspark.sql.functions, and type the literals of NumPy scalars by their dtype,
as PySpark does
### What changes were proposed in this pull request?
- `pyspark.sql.functions.__all__` is now official PySpark 4.3.0's static
list of 562 names, in its order, so `from pyspark.sql.functions import *` no
longer exports `chr`, `random`, `uuid`, `countDistinct`, `bitwiseNOT`,
`shiftLeft`, `shiftRight`, `shiftRightUnsigned`, `toDegrees`, `toRadians`,
`sumDistinct`, `approxCountDistinct` and the avro and protobuf functions; those
that official has stay attributes. The public attributes of the module are
official's: `from_avro`, `to_avro`, `f [...]
- `lit()`, and Column operations with a value, type a NumPy scalar by its
dtype, as official's `_from_numpy_type` does: `int8`/`int16`/`int32`/`int64` as
tinyint/smallint/int/bigint, `float32` as float, `float64` as double, `bool_`
as boolean; unsigned ints, `float16`, `longdouble` and complex numbers raise
`UNSUPPORTED_DATA_TYPE` (`crates/pyspark-rs/src/functions.rs`).
### Why are the changes needed?
[SPARK-61065](https://issues.apache.org/jira/browse/SPARK-61065): the star
import, common in notebooks, shadowed builtins:
| Code | Current master | This change and PySpark 4.3.0 |
| --- | --- | --- |
| `from pyspark.sql.functions import *; chr(65)` | `PySparkTypeError
[NOT_EXPECTED_TYPE]` | `'A'` |
| `import random; from pyspark.sql.functions import *; random.randint(1,
1)` | `AttributeError` | `1` |
| `hasattr(F, "from_avro")` | `True` | `False` |
| `spark.range(1).select(F.lit(np.int64(5))).schema` | `struct<5:int>` |
`struct<5:bigint>` |
| `F.lit(np.float32(1.5))` | double | float |
| `F.lit(np.uint8(5))` | int | `UNSUPPORTED_DATA_TYPE` `uint8` |
### Does this PR introduce _any_ user-facing change?
Yes. The star import and the module's attributes are official's, the avro
and protobuf functions are on their official modules, and NumPy scalars are
typed as in PySpark.
### How was this patch tested?
- New offline tests in `python/tests/test_dropin_offline.py` for the
exports, the avro and protobuf modules, and the literal types of each NumPy
scalar (from the plan), and a live test in `python/tests/test_dropin_e2e.py`;
`test_functions_exports_fresh_process` now compares with official's list.
- The offline suite (961 passed, 8 skipped); about 40 `lit` cases match
official's types, column names and errors.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #256 from HyukjinKwon/SPARK-61065-functions-exports.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/functions.rs | 52 +++
python/pyspark/sql/avro/functions.py | 66 ++-
python/pyspark/sql/functions.py | 727 +++++++++++++++++++++++++------
python/pyspark/sql/protobuf/functions.py | 85 +++-
python/tests/test_dropin_e2e.py | 9 +
python/tests/test_dropin_offline.py | 239 +++++++++-
6 files changed, 1035 insertions(+), 143 deletions(-)
diff --git a/crates/pyspark-rs/src/functions.rs
b/crates/pyspark-rs/src/functions.rs
index 31e1aca..3a2c0ed 100644
--- a/crates/pyspark-rs/src/functions.rs
+++ b/crates/pyspark-rs/src/functions.rs
@@ -224,6 +224,11 @@ pub fn to_column(obj: &Bound<'_, PyAny>) ->
PyResult<Column> {
));
}
+ // A NumPy scalar is typed by its dtype, as PySpark's
`LiteralExpression._infer_type`.
+ if let Some(lit) = numpy_scalar_literal(obj)? {
+ return Ok(Column::new(Expression::literal(lit)));
+ }
+
// Non-primitive scalars must be checked BEFORE int/float: Decimal defines
// __float__/__int__, and a pandas Timestamp (a datetime subclass) defines
__int__
// (nanoseconds), so an early int/float extract would silently mis-encode
them.
@@ -288,6 +293,53 @@ fn numpy_ndarray_ndim(obj: &Bound<'_, PyAny>) ->
PyResult<Option<usize>> {
}
}
+/// The literal of a NumPy scalar, typed as PySpark's
`LiteralExpression._infer_type` types it: a
+/// bool, a signed int of 8 to 64 bits (tinyint to bigint), or a float32 or
float64. A str or bytes
+/// scalar is `None`, for the string and binary conversions to take. Any other
NumPy scalar (an
+/// unsigned int, float16, longdouble, complex, datetime64 or timedelta64) is
+/// UNSUPPORTED_DATA_TYPE, named by its Python type. NumPy is looked up in
`sys.modules`, as a
+/// NumPy scalar exists only when NumPy is imported.
+fn numpy_scalar_literal(obj: &Bound<'_, PyAny>) ->
PyResult<Option<LiteralExpression>> {
+ let py = obj.py();
+ let modules = py.import("sys")?.getattr("modules")?;
+ let numpy = match
modules.cast::<pyo3::types::PyDict>()?.get_item("numpy")? {
+ Some(numpy) if !numpy.is_none() => numpy,
+ _ => return Ok(None),
+ };
+ if !obj.is_instance(&numpy.getattr("generic")?)? {
+ return Ok(None);
+ }
+ if obj.is_instance(&numpy.getattr("bool_")?)? {
+ return Ok(Some(LiteralExpression::Boolean(obj.is_truthy()?)));
+ }
+ let dtype = obj.getattr("dtype")?;
+ let kind: String = dtype.getattr("kind")?.extract()?;
+ let itemsize: usize = dtype.getattr("itemsize")?.extract()?;
+ let literal = match (kind.as_str(), itemsize) {
+ ("U", _) | ("S", _) => return Ok(None),
+ ("i", 1) => LiteralExpression::Byte(obj.extract::<i64>()? as i32),
+ ("i", 2) => LiteralExpression::Short(obj.extract::<i64>()? as i32),
+ ("i", 4) => LiteralExpression::Integer(obj.extract::<i64>()? as i32),
+ ("i", 8) => LiteralExpression::Long(obj.extract::<i64>()?),
+ ("f", 4) => LiteralExpression::Float(obj.extract::<f64>()? as f32),
+ ("f", 8) => LiteralExpression::Double(obj.extract::<f64>()?),
+ _ => {
+ let type_name = obj
+ .get_type()
+ .name()
+ .map(|n| n.to_string())
+ .unwrap_or_else(|_| "<unknown>".to_string());
+ return Err(crate::column::raise_pyspark(
+ py,
+ "PySparkTypeError",
+ "UNSUPPORTED_DATA_TYPE",
+ &[("data_type", &type_name)],
+ ));
+ }
+ };
+ Ok(Some(literal))
+}
+
/// Build a literal for the non-primitive scalar Python types (`datetime`,
`date`,
/// `decimal.Decimal`, `bytes`/`bytearray`). Returns `None` if `obj` is not
one of them,
/// so the caller falls through to the primitive int/float/str/bool checks.
Kept in sync
diff --git a/python/pyspark/sql/avro/functions.py
b/python/pyspark/sql/avro/functions.py
index 300abe3..2a568b6 100644
--- a/python/pyspark/sql/avro/functions.py
+++ b/python/pyspark/sql/avro/functions.py
@@ -14,9 +14,69 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-"""Avro functions, mirroring ``pyspark.sql.avro.functions`` (the
implementations live
-in ``pyspark.sql.functions``; re-exported here for the official import
path)."""
+"""Avro functions, mirroring ``pyspark.sql.avro.functions``."""
+
+from pyspark import _pyspark
+from pyspark.errors import PySparkTypeError
+from pyspark.sql.column import Column
+from pyspark.sql.functions import _call_function, _dict_to_options_column,
_to_col
+
+_functions = _pyspark.functions
+_pyfunc_from_avro = _functions.pyfunc_from_avro
+_pyfunc_from_avro_with_options = _functions.pyfunc_from_avro_with_options
+_pyfunc_to_avro_with_schema = _functions.pyfunc_to_avro_with_schema
+
+
+def _check_data(data):
+ if not isinstance(data, (Column, str)):
+ raise PySparkTypeError(
+ errorClass="INVALID_TYPE",
+ messageParameters={"arg_name": "data", "arg_type":
"pyspark.sql.Column or str"},
+ )
+
+
+def from_avro(data, jsonFormatSchema, options=None):
+ """Converts a binary column of Avro format into its corresponding catalyst
value.
+
+ Args:
+ data: the binary column (a Column or a column name).
+ jsonFormatSchema: the Avro schema in JSON string format.
+ options: optional dict of options for the Avro reader.
+ """
+ _check_data(data)
+ if not isinstance(jsonFormatSchema, str):
+ raise PySparkTypeError(
+ errorClass="INVALID_TYPE",
+ messageParameters={"arg_name": "jsonFormatSchema", "arg_type":
"str"},
+ )
+ if options is not None and not isinstance(options, dict):
+ raise PySparkTypeError(
+ errorClass="INVALID_TYPE",
+ messageParameters={"arg_name": "options", "arg_type": "dict,
optional"},
+ )
+ if options is None:
+ return _pyfunc_from_avro(_to_col(data), jsonFormatSchema)
+ return _pyfunc_from_avro_with_options(
+ _to_col(data), jsonFormatSchema, _dict_to_options_column(options)
+ )
+
+
+def to_avro(data, jsonFormatSchema=""):
+ """Converts a column into binary of Avro format.
+
+ Args:
+ data: the data column (a Column or a column name).
+ jsonFormatSchema: the Avro schema in JSON string format; empty to
infer it.
+ """
+ _check_data(data)
+ if not isinstance(jsonFormatSchema, str):
+ raise PySparkTypeError(
+ errorClass="INVALID_TYPE",
+ messageParameters={"arg_name": "jsonFormatSchema", "arg_type":
"str"},
+ )
+ if jsonFormatSchema == "":
+ return _call_function("to_avro", _to_col(data))
+ return _pyfunc_to_avro_with_schema(_to_col(data), jsonFormatSchema)
-from pyspark.sql.functions import from_avro, to_avro
__all__ = ["from_avro", "to_avro"]
diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py
index f362437..7e2034d 100644
--- a/python/pyspark/sql/functions.py
+++ b/python/pyspark/sql/functions.py
@@ -1,8 +1,40 @@
"""pyspark.sql.functions module exposing Rust-backed SQL functions."""
-import os
+import decimal
+import functools
+import importlib.util as _importlib_util
+import inspect
+import os as _os
+import sys
+import warnings
+from typing import (
+ TYPE_CHECKING,
+ Any,
+ Callable,
+ Iterable,
+ Mapping,
+ Optional,
+ Sequence,
+ Tuple,
+ Type,
+ Union,
+ ValuesView,
+ cast,
+ overload,
+)
from pyspark import _pyspark
+from pyspark.errors import PySparkTypeError, PySparkValueError
+from pyspark.sql.column import Column
+from pyspark.sql.types import (
+ ArrayType,
+ ByteType,
+ DataType,
+ MapType,
+ NumericType,
+ StringType,
+ StructType,
+)
_functions = getattr(_pyspark, 'functions', None)
@@ -16,18 +48,6 @@ _pyfunc_expr = _functions.pyfunc_expr
_pyfunc_count = _functions.pyfunc_count
_pyfunc_when = _functions.pyfunc_when
_call_function = _functions.pyfunc_call_function
-# Mixed function bindings
-_pyfunc_from_avro = _functions.pyfunc_from_avro
-_pyfunc_from_avro_with_options = _functions.pyfunc_from_avro_with_options
-_pyfunc_to_avro_with_schema = _functions.pyfunc_to_avro_with_schema
-_pyfunc_from_protobuf = _functions.pyfunc_from_protobuf
-_pyfunc_from_protobuf_with_descriptor =
_functions.pyfunc_from_protobuf_with_descriptor
-_pyfunc_from_protobuf_with_descriptor_and_options =
_functions.pyfunc_from_protobuf_with_descriptor_and_options
-_pyfunc_from_protobuf_with_options =
_functions.pyfunc_from_protobuf_with_options
-_pyfunc_to_protobuf = _functions.pyfunc_to_protobuf
-_pyfunc_to_protobuf_with_descriptor =
_functions.pyfunc_to_protobuf_with_descriptor
-_pyfunc_to_protobuf_with_descriptor_and_options =
_functions.pyfunc_to_protobuf_with_descriptor_and_options
-_pyfunc_to_protobuf_with_options = _functions.pyfunc_to_protobuf_with_options
# Utility functions for wrapping/unwrapping columns
def _unwrap(obj):
@@ -290,8 +310,8 @@ def _dict_to_options_column(options_dict):
# Convert dict to alternating keys and values: [k1, v1, k2, v2, ...]
args = []
for k, v in options_dict.items():
- args.append(k)
- args.append(v)
+ args.append(str(k))
+ args.append(str(v))
# Use create_map with literal strings - we'll build the Column through
_call_function
# since create_map is auto-generated
return _call_function("create_map", *[_pyfunc_lit(arg) for arg in
args])
@@ -380,94 +400,12 @@ def when(condition, value):
return _pyfunc_when(condition, value if isinstance(value, _pyspark.Column)
else lit(value))
-# Mixed/special functions - explicit wrappers for non-generic dispatch
-def from_avro(data, json_format_schema):
- """Deserialize Avro data into a column.
-
- Args:
- data: column containing binary Avro data
- json_format_schema: JSON string schema for the Avro data
- """
- return _pyfunc_from_avro(_to_col(data), json_format_schema)
-
-def from_avro_with_options(data, json_format_schema, options=None):
- """Deserialize Avro data into a column with options.
-
- Args:
- data: column containing binary Avro data
- json_format_schema: JSON string schema for the Avro data
- options: dict or Column of options
- """
- if options is None:
- return _pyfunc_from_avro(_to_col(data), json_format_schema)
- options_col = _dict_to_options_column(options)
- return _pyfunc_from_avro_with_options(_to_col(data), json_format_schema,
options_col)
-
-def to_avro_with_schema(data, json_format_schema):
- """Serialize a column to Avro binary format with schema.
-
- Args:
- data: column to serialize
- json_format_schema: JSON string schema for the Avro data
- """
- return _pyfunc_to_avro_with_schema(_to_col(data), json_format_schema)
-
-def from_protobuf(data, message_name, desc_file_path=None, options=None,
binary_descriptor_set=None):
- """Deserialize Protobuf data into a column.
-
- Args:
- data: column containing binary Protobuf data
- message_name: name of the Protobuf message type
- desc_file_path: optional, path to descriptor file
- options: optional, dict or Column of options
- binary_descriptor_set: optional, binary descriptor set bytes
- """
- if binary_descriptor_set is not None and options is not None:
- options_col = _dict_to_options_column(options)
- return _pyfunc_from_protobuf_with_descriptor_and_options(
- _to_col(data), message_name, binary_descriptor_set, options_col
- )
- elif binary_descriptor_set is not None:
- return _pyfunc_from_protobuf_with_descriptor(
- _to_col(data), message_name, binary_descriptor_set
- )
- elif options is not None:
- options_col = _dict_to_options_column(options)
- return _pyfunc_from_protobuf_with_options(_to_col(data), message_name,
options_col)
- else:
- return _pyfunc_from_protobuf(_to_col(data), message_name)
-
-def to_protobuf(data, message_name, desc_file_path=None, options=None,
binary_descriptor_set=None):
- """Serialize a column to Protobuf binary format.
-
- Args:
- data: column to serialize
- message_name: name of the Protobuf message type
- desc_file_path: optional, path to descriptor file
- options: optional, dict or Column of options
- binary_descriptor_set: optional, binary descriptor set bytes
- """
- if binary_descriptor_set is not None and options is not None:
- options_col = _dict_to_options_column(options)
- return _pyfunc_to_protobuf_with_descriptor_and_options(
- _to_col(data), message_name, binary_descriptor_set, options_col
- )
- elif binary_descriptor_set is not None:
- return _pyfunc_to_protobuf_with_descriptor(
- _to_col(data), message_name, binary_descriptor_set
- )
- elif options is not None:
- options_col = _dict_to_options_column(options)
- return _pyfunc_to_protobuf_with_options(_to_col(data), message_name,
options_col)
- else:
- return _pyfunc_to_protobuf(_to_col(data), message_name)
-
# Now we can import generated wrappers (they use _create_wrapper defined above)
# We do this via exec to make _create_wrapper available in the generated
module's namespace
-import importlib.util
-spec = importlib.util.spec_from_file_location("functions_generated",
-
os.path.join(os.path.dirname(__file__), "functions_generated.py"))
-_gen_module = importlib.util.module_from_spec(spec)
+_spec = _importlib_util.spec_from_file_location(
+ "functions_generated", _os.path.join(_os.path.dirname(__file__),
"functions_generated.py")
+)
+_gen_module = _importlib_util.module_from_spec(_spec)
# Inject our helper functions into the generated module's namespace before
loading. The
# functions generated from their PySpark Connect implementations call lit,
column and
# _invoke_function (an UnresolvedFunction of the name and arguments), as
PySpark Connect's
@@ -481,11 +419,14 @@ _gen_module._call_function = _call_function
_gen_module.lit = lit
_gen_module.column = _pyfunc_col
_gen_module._invoke_function = _functions.pyfunc_invoke_function
-spec.loader.exec_module(_gen_module)
+_spec.loader.exec_module(_gen_module)
-# Import all generated functions into this module
+# Import all generated functions into this module, except those that PySpark
does not expose
+# here: to_avro is pyspark.sql.avro.functions.to_avro, which takes the schema
of its data.
+_NOT_IN_PYSPARK_FUNCTIONS = ("to_avro",)
for name in _gen_module.__all__:
- globals()[name] = getattr(_gen_module, name)
+ if name not in _NOT_IN_PYSPARK_FUNCTIONS:
+ globals()[name] = getattr(_gen_module, name)
# PySpark Connect's _to_col, which raises PySparkTypeError NOT_EXPECTED_TYPE
for an
# argument that is neither a Column nor a column name.
@@ -808,34 +749,576 @@ except Exception: # pragma: no cover
pass
from pyspark.sql.pandas.functions import PandasUDFType, ArrowUDFType # noqa:
E402,F401
-# Build __all__ with all function names
+# The public names of PySpark 4.3.0's pyspark.sql.functions, in its order. A
star import
+# brings in exactly these; the other names of this module (e.g. chr, random,
uuid, which
+# PySpark also has as attributes) are not exported, so they do not shadow the
builtins and
+# the stdlib modules of the same names.
__all__ = [
+ "broadcast",
+ "call_function",
"col",
+ "column",
"lit",
"expr",
- "sum",
- "count",
+ "coalesce",
+ "ifnull",
+ "nanvl",
+ "nullif",
+ "nullifzero",
+ "nvl",
+ "nvl2",
+ "when",
+ "zeroifnull",
+ "equal_null",
+ "ilike",
+ "isnan",
+ "isnotnull",
+ "isnull",
+ "like",
+ "regexp",
+ "regexp_like",
+ "rlike",
+ "asc",
+ "asc_nulls_first",
+ "asc_nulls_last",
+ "desc",
+ "desc_nulls_first",
+ "desc_nulls_last",
+ "abs",
+ "acos",
+ "acosh",
+ "asin",
+ "asinh",
+ "atan",
+ "atan2",
+ "atanh",
+ "bin",
+ "bround",
+ "cbrt",
+ "ceil",
+ "ceiling",
+ "conv",
+ "cos",
+ "cosh",
+ "cot",
+ "csc",
+ "degrees",
+ "e",
+ "exp",
+ "expm1",
+ "factorial",
+ "floor",
+ "greatest",
+ "hex",
+ "hypot",
+ "least",
+ "ln",
+ "log",
+ "log10",
+ "log1p",
+ "log2",
+ "negate",
+ "negative",
+ "pi",
+ "pmod",
+ "positive",
+ "pow",
+ "power",
+ "radians",
+ "rand",
+ "randn",
+ "rint",
+ "round",
+ "sec",
+ "sign",
+ "signum",
+ "sin",
+ "sinh",
+ "sqrt",
+ "tan",
+ "tanh",
+ "try_add",
+ "try_divide",
+ "try_mod",
+ "try_multiply",
+ "try_subtract",
+ "unhex",
+ "uniform",
+ "width_bucket",
+ "ascii",
+ "base64",
+ "bit_length",
+ "btrim",
+ "char",
+ "char_length",
+ "character_length",
+ "collate",
+ "collation",
+ "concat_ws",
+ "contains",
+ "decode",
+ "elt",
+ "encode",
+ "endswith",
+ "find_in_set",
+ "format_number",
+ "format_string",
+ "from_base32",
+ "initcap",
+ "instr",
+ "is_valid_utf8",
+ "jaro_winkler_similarity",
+ "lcase",
+ "left",
+ "length",
+ "levenshtein",
+ "locate",
+ "lower",
+ "lpad",
+ "ltrim",
+ "make_valid_utf8",
+ "mask",
+ "octet_length",
+ "overlay",
+ "position",
+ "printf",
+ "quote",
+ "randstr",
+ "regexp_count",
+ "regexp_extract",
+ "regexp_extract_all",
+ "regexp_instr",
+ "regexp_replace",
+ "regexp_substr",
+ "repeat",
+ "replace",
+ "right",
+ "rpad",
+ "rtrim",
+ "sentences",
+ "soundex",
+ "split",
+ "split_part",
+ "startswith",
+ "substr",
+ "substring",
+ "substring_index",
+ "to_base32",
+ "to_binary",
+ "to_char",
+ "to_number",
+ "to_varchar",
+ "translate",
+ "trim",
+ "try_to_binary",
+ "try_to_number",
+ "try_validate_utf8",
+ "ucase",
+ "unbase64",
+ "upper",
+ "validate_utf8",
+ "bit_count",
+ "bit_get",
+ "bitwise_not",
+ "getbit",
+ "shiftleft",
+ "shiftright",
+ "shiftrightunsigned",
+ "add_months",
+ "convert_timezone",
+ "curdate",
+ "current_date",
+ "current_time",
+ "current_timestamp",
+ "current_timezone",
+ "date_add",
+ "date_diff",
+ "date_format",
+ "date_from_unix_date",
+ "date_part",
+ "date_sub",
+ "date_trunc",
+ "dateadd",
+ "datediff",
+ "datepart",
+ "day",
+ "dayname",
+ "dayofmonth",
+ "dayofweek",
+ "dayofyear",
+ "extract",
+ "from_unixtime",
+ "from_utc_timestamp",
+ "hour",
+ "last_day",
+ "localtimestamp",
+ "make_date",
+ "make_dt_interval",
+ "make_interval",
+ "make_time",
+ "make_timestamp",
+ "make_timestamp_ltz",
+ "make_timestamp_ntz",
+ "make_ym_interval",
+ "minute",
+ "month",
+ "monthname",
+ "months_between",
+ "next_day",
+ "now",
+ "quarter",
+ "second",
+ "session_window",
+ "timestamp_add",
+ "timestamp_diff",
+ "timestamp_micros",
+ "timestamp_millis",
+ "timestamp_nanos",
+ "timestamp_seconds",
+ "time_bucket",
+ "time_diff",
+ "time_from_micros",
+ "time_from_millis",
+ "time_from_seconds",
+ "time_to_micros",
+ "time_to_millis",
+ "time_to_seconds",
+ "time_trunc",
+ "to_date",
+ "to_time",
+ "to_timestamp",
+ "to_timestamp_ltz",
+ "to_timestamp_ntz",
+ "to_unix_timestamp",
+ "to_utc_timestamp",
+ "trunc",
+ "try_make_interval",
+ "try_make_timestamp",
+ "try_make_timestamp_ltz",
+ "try_make_timestamp_ntz",
+ "try_to_time",
+ "try_to_timestamp",
+ "try_to_date",
+ "unix_date",
+ "unix_micros",
+ "unix_millis",
+ "unix_nanos",
+ "unix_seconds",
+ "unix_timestamp",
+ "weekday",
+ "weekofyear",
+ "window",
+ "window_time",
+ "year",
+ "crc32",
+ "hash",
+ "md5",
+ "sha",
+ "sha1",
+ "sha2",
+ "xxhash64",
+ "aggregate",
+ "array_sort",
+ "cardinality",
+ "concat",
+ "element_at",
+ "exists",
+ "filter",
+ "forall",
+ "map_filter",
+ "map_zip_with",
+ "reduce",
+ "reverse",
+ "size",
+ "transform",
+ "transform_keys",
+ "transform_values",
+ "try_element_at",
+ "zip_with",
+ "array",
+ "array_append",
+ "array_compact",
+ "array_contains",
+ "array_distinct",
+ "array_except",
+ "array_insert",
+ "array_intersect",
+ "array_join",
+ "array_max",
+ "array_min",
+ "array_position",
+ "array_prepend",
+ "array_remove",
+ "array_repeat",
+ "array_size",
+ "array_union",
+ "arrays_overlap",
+ "arrays_zip",
+ "flatten",
+ "get",
+ "sequence",
+ "shuffle",
+ "slice",
+ "sort_array",
+ "named_struct",
+ "struct",
+ "create_map",
+ "map_concat",
+ "map_contains_key",
+ "map_entries",
+ "map_from_arrays",
+ "map_from_entries",
+ "map_keys",
+ "map_values",
+ "str_to_map",
+ "any_value",
+ "approx_count_distinct",
+ "approx_percentile",
+ "array_agg",
"avg",
+ "bit_and",
+ "bit_or",
+ "bit_xor",
+ "bitmap_and_agg",
+ "bitmap_construct_agg",
+ "bitmap_or_agg",
+ "bool_and",
+ "bool_or",
+ "collect_list",
+ "collect_set",
+ "collect_union",
+ "corr",
+ "count",
+ "count_distinct",
+ "count_if",
+ "count_min_sketch",
+ "covar_pop",
+ "covar_samp",
+ "every",
+ "first",
+ "first_value",
+ "grouping",
+ "grouping_id",
+ "histogram_numeric",
+ "hll_sketch_agg",
+ "hll_union_agg",
+ "kll_sketch_agg_bigint",
+ "kll_sketch_agg_double",
+ "kll_sketch_agg_float",
+ "kll_merge_agg_bigint",
+ "kll_merge_agg_double",
+ "kll_merge_agg_float",
+ "kurtosis",
+ "last",
+ "last_value",
+ "listagg",
+ "listagg_distinct",
"max",
+ "max_by",
+ "mean",
+ "median",
"min",
- "when",
- "udf",
+ "min_by",
+ "mode",
+ "percentile",
+ "percentile_approx",
+ "product",
+ "regr_avgx",
+ "regr_avgy",
+ "regr_count",
+ "regr_intercept",
+ "regr_r2",
+ "regr_slope",
+ "regr_sxx",
+ "regr_sxy",
+ "regr_syy",
+ "skewness",
+ "some",
+ "std",
+ "stddev",
+ "stddev_pop",
+ "stddev_samp",
+ "string_agg",
+ "string_agg_distinct",
+ "sum",
+ "sum_distinct",
+ "theta_intersection_agg",
+ "theta_sketch_agg",
+ "theta_union_agg",
+ "tuple_intersection_agg_double",
+ "tuple_intersection_agg_integer",
+ "tuple_sketch_agg_double",
+ "tuple_sketch_agg_integer",
+ "tuple_union_agg_double",
+ "tuple_union_agg_integer",
+ "try_avg",
+ "try_sum",
+ "var_pop",
+ "var_samp",
+ "variance",
+ "counter_diff",
+ "cume_dist",
+ "dense_rank",
+ "lag",
+ "lead",
+ "nth_value",
+ "ntile",
+ "percent_rank",
+ "rank",
+ "row_number",
+ "explode",
+ "explode_outer",
+ "inline",
+ "inline_outer",
+ "posexplode",
+ "posexplode_outer",
+ "stack",
+ "years",
+ "months",
+ "days",
+ "hours",
+ "bucket",
+ "from_csv",
+ "schema_of_csv",
+ "to_csv",
+ "from_json",
+ "get_json_object",
+ "json_array_length",
+ "json_object_keys",
+ "json_tuple",
+ "schema_of_json",
+ "to_json",
+ "is_valid_variant",
+ "is_variant_null",
+ "parse_json",
+ "schema_of_variant",
+ "schema_of_variant_agg",
+ "try_variant_get",
+ "variant_array_append",
+ "try_variant_array_append",
+ "variant_delete",
+ "variant_strip_nulls",
+ "variant_get",
+ "variant_insert",
+ "try_variant_insert",
+ "variant_set",
+ "try_variant_set",
+ "try_parse_json",
+ "to_variant_object",
+ "from_xml",
+ "schema_of_xml",
+ "to_xml",
+ "xpath",
+ "xpath_boolean",
+ "xpath_double",
+ "xpath_float",
+ "xpath_int",
+ "xpath_long",
+ "xpath_number",
+ "xpath_short",
+ "xpath_string",
+ "parse_url",
+ "try_parse_url",
+ "url_decode",
+ "url_encode",
+ "try_url_decode",
+ "aes_decrypt",
+ "aes_encrypt",
+ "assert_true",
+ "bitmap_bit_position",
+ "bitmap_bucket_number",
+ "bitmap_count",
+ "current_catalog",
+ "current_database",
+ "current_path",
+ "current_schema",
+ "current_user",
+ "hmac",
+ "input_file_block_length",
+ "input_file_block_start",
+ "input_file_name",
+ "java_method",
+ "monotonically_increasing_id",
+ "raise_error",
+ "reflect",
+ "session_user",
+ "spark_partition_id",
+ "try_aes_decrypt",
+ "try_reflect",
+ "typeof",
+ "user",
+ "version",
+ "AnalyzeArgument",
+ "AnalyzeResult",
+ "ArrowUDFType",
+ "OrderingColumn",
+ "PandasUDFType",
+ "PartitioningColumn",
+ "SelectedColumn",
+ "SkipRestOfInputTableException",
+ "UserDefinedFunction",
+ "UserDefinedTableFunction",
+ "arrow_udf",
+ "hll_sketch_estimate",
+ "hll_union",
+ "kll_sketch_get_n_bigint",
+ "kll_sketch_get_n_double",
+ "kll_sketch_get_n_float",
+ "kll_sketch_get_quantile_bigint",
+ "kll_sketch_get_quantile_double",
+ "kll_sketch_get_quantile_float",
+ "kll_sketch_get_rank_bigint",
+ "kll_sketch_get_rank_double",
+ "kll_sketch_get_rank_float",
+ "kll_sketch_merge_bigint",
+ "kll_sketch_merge_double",
+ "kll_sketch_merge_float",
+ "kll_sketch_to_string_bigint",
+ "kll_sketch_to_string_double",
+ "kll_sketch_to_string_float",
+ "theta_difference",
+ "theta_intersection",
+ "theta_sketch_estimate",
+ "theta_union",
+ "tuple_difference_double",
+ "tuple_difference_integer",
+ "tuple_difference_theta_double",
+ "tuple_difference_theta_integer",
+ "tuple_intersection_double",
+ "tuple_intersection_integer",
+ "tuple_intersection_theta_double",
+ "tuple_intersection_theta_integer",
+ "tuple_sketch_estimate_double",
+ "tuple_sketch_estimate_integer",
+ "tuple_sketch_summary_double",
+ "tuple_sketch_summary_integer",
+ "tuple_sketch_theta_double",
+ "tuple_sketch_theta_integer",
+ "tuple_union_double",
+ "tuple_union_integer",
+ "tuple_union_theta_double",
+ "tuple_union_theta_integer",
+ "st_asbinary",
+ "st_geogfromwkb",
+ "st_geomfromwkb",
+ "st_setsrid",
+ "st_srid",
+ "vector_cosine_similarity",
+ "vector_inner_product",
+ "vector_l2_distance",
+ "vector_norm",
+ "vector_normalize",
+ "vector_avg",
+ "vector_sum",
+ "call_udf",
"pandas_udf",
+ "udf",
+ "udtf",
+ "arrow_udtf",
+ "unwrap_udt",
]
-
-# Add all other function names (dynamically)
-import inspect
-_export_globals = list(globals().items())
-_exported_names = set(__all__)
-for name, obj in _export_globals:
- if (callable(obj) and
- not name.startswith('_') and
- name not in _exported_names and
- name not in ['inspect', 'importlib', '_gen_module', 'spec', 'os']):
- __all__.append(name)
- _exported_names.add(name)
# Leave no loop variables behind, as PySpark has no `functions.name` or
`functions.obj`.
-del _export_globals, _exported_names, name, obj
+del name
# Expose the `builtin` submodule at ``pyspark.sql.functions.builtin``.
Upstream v4.2.0 splits
# functions into a package whose ``builtin`` submodule holds the actual
functions and whose
diff --git a/python/pyspark/sql/protobuf/functions.py
b/python/pyspark/sql/protobuf/functions.py
index dfd4f85..a389602 100644
--- a/python/pyspark/sql/protobuf/functions.py
+++ b/python/pyspark/sql/protobuf/functions.py
@@ -14,9 +14,88 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-"""Protobuf functions, mirroring ``pyspark.sql.protobuf.functions``
(implementations
-live in ``pyspark.sql.functions``; re-exported here for the official import
path)."""
+"""Protobuf functions, mirroring ``pyspark.sql.protobuf.functions``."""
+
+from pyspark import _pyspark
+from pyspark.sql.functions import _dict_to_options_column, _to_col
+
+_functions = _pyspark.functions
+_pyfunc_from_protobuf = _functions.pyfunc_from_protobuf
+_pyfunc_from_protobuf_with_descriptor =
_functions.pyfunc_from_protobuf_with_descriptor
+_pyfunc_from_protobuf_with_descriptor_and_options = (
+ _functions.pyfunc_from_protobuf_with_descriptor_and_options
+)
+_pyfunc_from_protobuf_with_options =
_functions.pyfunc_from_protobuf_with_options
+_pyfunc_to_protobuf = _functions.pyfunc_to_protobuf
+_pyfunc_to_protobuf_with_descriptor =
_functions.pyfunc_to_protobuf_with_descriptor
+_pyfunc_to_protobuf_with_descriptor_and_options = (
+ _functions.pyfunc_to_protobuf_with_descriptor_and_options
+)
+_pyfunc_to_protobuf_with_options = _functions.pyfunc_to_protobuf_with_options
+
+
+def _descriptor_set(descFilePath, binaryDescriptorSet):
+ """The binary descriptor set: the given bytes, else the content of the
descriptor file."""
+ if binaryDescriptorSet is not None:
+ return binaryDescriptorSet
+ if descFilePath is not None:
+ with open(descFilePath, "rb") as f:
+ return f.read()
+ return None
+
+
+def from_protobuf(
+ data, messageName, descFilePath=None, options=None,
binaryDescriptorSet=None
+):
+ """Converts a binary column of Protobuf format into its corresponding
catalyst value.
+
+ Args:
+ data: the binary column (a Column or a column name).
+ messageName: the protobuf message name to look for in the descriptor
file.
+ descFilePath: the protobuf descriptor file path.
+ options: optional dict of options for the Protobuf reader.
+ binaryDescriptorSet: the protobuf descriptor set as bytes, instead of
a file.
+ """
+ descriptor = _descriptor_set(descFilePath, binaryDescriptorSet)
+ column = _to_col(data)
+ if descriptor is None:
+ if options is None:
+ return _pyfunc_from_protobuf(column, messageName)
+ return _pyfunc_from_protobuf_with_options(
+ column, messageName, _dict_to_options_column(options)
+ )
+ if options is None:
+ return _pyfunc_from_protobuf_with_descriptor(column, messageName,
descriptor)
+ return _pyfunc_from_protobuf_with_descriptor_and_options(
+ column, messageName, descriptor, _dict_to_options_column(options)
+ )
+
+
+def to_protobuf(
+ data, messageName, descFilePath=None, options=None,
binaryDescriptorSet=None
+):
+ """Converts a column into binary of Protobuf format.
+
+ Args:
+ data: the data column (a Column or a column name).
+ messageName: the protobuf message name to look for in the descriptor
file.
+ descFilePath: the protobuf descriptor file path.
+ options: optional dict of options for the Protobuf writer.
+ binaryDescriptorSet: the protobuf descriptor set as bytes, instead of
a file.
+ """
+ descriptor = _descriptor_set(descFilePath, binaryDescriptorSet)
+ column = _to_col(data)
+ if descriptor is None:
+ if options is None:
+ return _pyfunc_to_protobuf(column, messageName)
+ return _pyfunc_to_protobuf_with_options(
+ column, messageName, _dict_to_options_column(options)
+ )
+ if options is None:
+ return _pyfunc_to_protobuf_with_descriptor(column, messageName,
descriptor)
+ return _pyfunc_to_protobuf_with_descriptor_and_options(
+ column, messageName, descriptor, _dict_to_options_column(options)
+ )
-from pyspark.sql.functions import from_protobuf, to_protobuf
__all__ = ["from_protobuf", "to_protobuf"]
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index deb3ef3..a2efb5e 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -104,6 +104,15 @@ def test_conf_get_with_a_none_default(session):
session.conf.get(key)
+def
test_numpy_scalar_literals_have_the_types_of_their_dtypes_as_pyspark(session):
+ np = pytest.importorskip("numpy")
+ from pyspark.sql import functions as F
+
+ # As PySpark: a NumPy scalar's literal is typed by its dtype, as the
server reports it.
+ df = session.range(1).select(F.lit(np.int64(5)), F.lit(np.float32(1.5)),
F.lit(np.int8(5)))
+ assert [f.dataType.simpleString() for f in df.schema.fields] == ["bigint",
"float", "tinyint"]
+
+
def test_user_id_of_the_connection_string_reaches_every_request_as_pyspark():
# The server keys a session by the user ID in each request. As in PySpark,
the
# queries carry the user ID of the connection string, as the configs do,
so a config
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 7e531bd..9d4bd24 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -560,12 +560,11 @@ def test_functions_exports_fresh_process():
from pyspark.sql import functions as F
assert not hasattr(F, "name") and not hasattr(F, "obj")
- initial = ["col", "lit", "expr", "sum", "count", "avg", "max", "min",
- "when", "udf", "pandas_udf"]
+ official = __OFFICIAL_FUNCTIONS_ALL__
def check_exports():
assert type(F.__all__) is list
- assert F.__all__[:len(initial)] == initial
+ assert F.__all__ == official
assert len(F.__all__) == len(set(F.__all__))
assert all(callable(getattr(F, name)) for name in F.__all__)
assert not any(name.startswith("_") for name in F.__all__)
@@ -605,7 +604,7 @@ def test_functions_exports_fresh_process():
importlib.reload(F)
assert F.__all__ is not old_all
assert check_exports() == expected
- ''')], capture_output=True, text=True)
+ ''').replace("__OFFICIAL_FUNCTIONS_ALL__", repr(OFFICIAL_FUNCTIONS_ALL))],
capture_output=True, text=True)
assert result.returncode == 0, result.stdout + result.stderr
@@ -1294,11 +1293,14 @@ def test_functions_module_wrappers():
assert type(F.sha2(c, 256)).__name__ == "Column"
assert type(F.window(F.col("t"), "10 minutes")).__name__ == "Column"
assert type(F.window(F.col("t"), "10 minutes", "5 minutes", "0
seconds")).__name__ == "Column"
- assert type(F.from_avro(c, "{}")).__name__ == "Column"
- assert type(F.from_avro_with_options(c, "{}", {"mode":
"PERMISSIVE"})).__name__ == "Column"
- assert type(F.to_avro_with_schema(c, "{}")).__name__ == "Column"
- assert type(F.from_protobuf(c, "M", options={"x": "y"})).__name__ ==
"Column"
- assert type(F.to_protobuf(c, "M", options={"x": "y"})).__name__ == "Column"
+ import pyspark.sql.avro.functions as avro
+ import pyspark.sql.protobuf.functions as protobuf
+
+ assert type(avro.from_avro(c, "{}")).__name__ == "Column"
+ assert type(avro.from_avro(c, "{}", {"mode": "PERMISSIVE"})).__name__ ==
"Column"
+ assert type(avro.to_avro(c, "{}")).__name__ == "Column"
+ assert type(protobuf.from_protobuf(c, "M", options={"x": "y"})).__name__
== "Column"
+ assert type(protobuf.to_protobuf(c, "M", options={"x": "y"})).__name__ ==
"Column"
def test_type_class_hierarchy():
@@ -2223,14 +2225,20 @@ def test_sql_is_remote():
assert S.is_remote() is True
-def test_functions_protobuf_descriptor_branches():
+def test_functions_protobuf_descriptor_branches(tmp_path):
+ import pyspark.sql.protobuf.functions as protobuf
from pyspark.sql import functions as F
+
c = F.col("v")
- # descriptor-file and binary-descriptor-set branches
- assert type(F.from_protobuf(c, "M",
desc_file_path="/tmp/x.desc")).__name__ == "Column"
- assert type(F.from_protobuf(c, "M",
binary_descriptor_set=b"\x00")).__name__ == "Column"
- assert type(F.to_protobuf(c, "M", desc_file_path="/tmp/x.desc")).__name__
== "Column"
- assert type(F.to_protobuf(c, "M", binary_descriptor_set=b"\x00")).__name__
== "Column"
+ descriptor = tmp_path / "m.desc"
+ descriptor.write_bytes(b"\x00")
+ # descriptor-file and binary-descriptor-set branches, with and without
options
+ assert type(protobuf.from_protobuf(c, "M",
descFilePath=str(descriptor))).__name__ == "Column"
+ assert type(protobuf.from_protobuf(c, "M",
binaryDescriptorSet=b"\x00")).__name__ == "Column"
+ assert type(protobuf.to_protobuf(c, "M",
descFilePath=str(descriptor))).__name__ == "Column"
+ assert type(protobuf.to_protobuf(c, "M",
binaryDescriptorSet=b"\x00")).__name__ == "Column"
+ assert type(protobuf.from_protobuf(c, "M", descFilePath=str(descriptor),
options={"x": "y"})).__name__ == "Column"
+ assert type(protobuf.to_protobuf(c, "M", binaryDescriptorSet=b"\x00",
options={"x": "y"})).__name__ == "Column"
def test_udf_defaults_and_registration():
@@ -7180,6 +7188,207 @@ def
test_completion_propagates_schema_errors_and_can_retry(completion_analyzed,
assert service.calls == 3
+# ---------------------------------------------------------------------------
functions exports and literals of NumPy scalars
+
+# The public names of PySpark 4.3.0's pyspark.sql.functions, in its order
(__all__ is static there).
+OFFICIAL_FUNCTIONS_ALL = """
+ broadcast call_function col column lit expr coalesce ifnull nanvl nullif
nullifzero nvl
+ nvl2 when zeroifnull equal_null ilike isnan isnotnull isnull like regexp
regexp_like
+ rlike asc asc_nulls_first asc_nulls_last desc desc_nulls_first
desc_nulls_last abs acos
+ acosh asin asinh atan atan2 atanh bin bround cbrt ceil ceiling conv cos
cosh cot csc
+ degrees e exp expm1 factorial floor greatest hex hypot least ln log log10
log1p log2
+ negate negative pi pmod positive pow power radians rand randn rint round
sec sign signum
+ sin sinh sqrt tan tanh try_add try_divide try_mod try_multiply
try_subtract unhex
+ uniform width_bucket ascii base64 bit_length btrim char char_length
character_length
+ collate collation concat_ws contains decode elt encode endswith find_in_set
+ format_number format_string from_base32 initcap instr is_valid_utf8
+ jaro_winkler_similarity lcase left length levenshtein locate lower lpad
ltrim
+ make_valid_utf8 mask octet_length overlay position printf quote randstr
regexp_count
+ regexp_extract regexp_extract_all regexp_instr regexp_replace
regexp_substr repeat
+ replace right rpad rtrim sentences soundex split split_part startswith
substr substring
+ substring_index to_base32 to_binary to_char to_number to_varchar translate
trim
+ try_to_binary try_to_number try_validate_utf8 ucase unbase64 upper
validate_utf8
+ bit_count bit_get bitwise_not getbit shiftleft shiftright
shiftrightunsigned add_months
+ convert_timezone curdate current_date current_time current_timestamp
current_timezone
+ date_add date_diff date_format date_from_unix_date date_part date_sub
date_trunc dateadd
+ datediff datepart day dayname dayofmonth dayofweek dayofyear extract
from_unixtime
+ from_utc_timestamp hour last_day localtimestamp make_date make_dt_interval
make_interval
+ make_time make_timestamp make_timestamp_ltz make_timestamp_ntz
make_ym_interval minute
+ month monthname months_between next_day now quarter second session_window
timestamp_add
+ timestamp_diff timestamp_micros timestamp_millis timestamp_nanos
timestamp_seconds
+ time_bucket time_diff time_from_micros time_from_millis time_from_seconds
time_to_micros
+ time_to_millis time_to_seconds time_trunc to_date to_time to_timestamp
to_timestamp_ltz
+ to_timestamp_ntz to_unix_timestamp to_utc_timestamp trunc try_make_interval
+ try_make_timestamp try_make_timestamp_ltz try_make_timestamp_ntz
try_to_time
+ try_to_timestamp try_to_date unix_date unix_micros unix_millis unix_nanos
unix_seconds
+ unix_timestamp weekday weekofyear window window_time year crc32 hash md5
sha sha1 sha2
+ xxhash64 aggregate array_sort cardinality concat element_at exists filter
forall
+ map_filter map_zip_with reduce reverse size transform transform_keys
transform_values
+ try_element_at zip_with array array_append array_compact array_contains
array_distinct
+ array_except array_insert array_intersect array_join array_max array_min
array_position
+ array_prepend array_remove array_repeat array_size array_union
arrays_overlap arrays_zip
+ flatten get sequence shuffle slice sort_array named_struct struct
create_map map_concat
+ map_contains_key map_entries map_from_arrays map_from_entries map_keys
map_values
+ str_to_map any_value approx_count_distinct approx_percentile array_agg avg
bit_and
+ bit_or bit_xor bitmap_and_agg bitmap_construct_agg bitmap_or_agg bool_and
bool_or
+ collect_list collect_set collect_union corr count count_distinct count_if
+ count_min_sketch covar_pop covar_samp every first first_value grouping
grouping_id
+ histogram_numeric hll_sketch_agg hll_union_agg kll_sketch_agg_bigint
+ kll_sketch_agg_double kll_sketch_agg_float kll_merge_agg_bigint
kll_merge_agg_double
+ kll_merge_agg_float kurtosis last last_value listagg listagg_distinct max
max_by mean
+ median min min_by mode percentile percentile_approx product regr_avgx
regr_avgy
+ regr_count regr_intercept regr_r2 regr_slope regr_sxx regr_sxy regr_syy
skewness some
+ std stddev stddev_pop stddev_samp string_agg string_agg_distinct sum
sum_distinct
+ theta_intersection_agg theta_sketch_agg theta_union_agg
tuple_intersection_agg_double
+ tuple_intersection_agg_integer tuple_sketch_agg_double
tuple_sketch_agg_integer
+ tuple_union_agg_double tuple_union_agg_integer try_avg try_sum var_pop
var_samp variance
+ counter_diff cume_dist dense_rank lag lead nth_value ntile percent_rank
rank row_number
+ explode explode_outer inline inline_outer posexplode posexplode_outer
stack years months
+ days hours bucket from_csv schema_of_csv to_csv from_json get_json_object
+ json_array_length json_object_keys json_tuple schema_of_json to_json
is_valid_variant
+ is_variant_null parse_json schema_of_variant schema_of_variant_agg
try_variant_get
+ variant_array_append try_variant_array_append variant_delete
variant_strip_nulls
+ variant_get variant_insert try_variant_insert variant_set try_variant_set
try_parse_json
+ to_variant_object from_xml schema_of_xml to_xml xpath xpath_boolean
xpath_double
+ xpath_float xpath_int xpath_long xpath_number xpath_short xpath_string
parse_url
+ try_parse_url url_decode url_encode try_url_decode aes_decrypt aes_encrypt
assert_true
+ bitmap_bit_position bitmap_bucket_number bitmap_count current_catalog
current_database
+ current_path current_schema current_user hmac input_file_block_length
+ input_file_block_start input_file_name java_method
monotonically_increasing_id
+ raise_error reflect session_user spark_partition_id try_aes_decrypt
try_reflect typeof
+ user version AnalyzeArgument AnalyzeResult ArrowUDFType OrderingColumn
PandasUDFType
+ PartitioningColumn SelectedColumn SkipRestOfInputTableException
UserDefinedFunction
+ UserDefinedTableFunction arrow_udf hll_sketch_estimate hll_union
kll_sketch_get_n_bigint
+ kll_sketch_get_n_double kll_sketch_get_n_float
kll_sketch_get_quantile_bigint
+ kll_sketch_get_quantile_double kll_sketch_get_quantile_float
kll_sketch_get_rank_bigint
+ kll_sketch_get_rank_double kll_sketch_get_rank_float
kll_sketch_merge_bigint
+ kll_sketch_merge_double kll_sketch_merge_float kll_sketch_to_string_bigint
+ kll_sketch_to_string_double kll_sketch_to_string_float theta_difference
+ theta_intersection theta_sketch_estimate theta_union
tuple_difference_double
+ tuple_difference_integer tuple_difference_theta_double
tuple_difference_theta_integer
+ tuple_intersection_double tuple_intersection_integer
tuple_intersection_theta_double
+ tuple_intersection_theta_integer tuple_sketch_estimate_double
+ tuple_sketch_estimate_integer tuple_sketch_summary_double
tuple_sketch_summary_integer
+ tuple_sketch_theta_double tuple_sketch_theta_integer tuple_union_double
+ tuple_union_integer tuple_union_theta_double tuple_union_theta_integer
st_asbinary
+ st_geogfromwkb st_geomfromwkb st_setsrid st_srid vector_cosine_similarity
+ vector_inner_product vector_l2_distance vector_norm vector_normalize
vector_avg
+ vector_sum call_udf pandas_udf udf udtf arrow_udtf unwrap_udt
+""".split()
+
+
+def test_star_import_of_functions_exports_the_official_names_only():
+ # A star import brings in exactly the names of PySpark's __all__. The
names that PySpark has
+ # only as attributes (chr, random, uuid, ...) are not exported, so they do
not shadow the
+ # builtin chr and the stdlib random and uuid; nor are the avro and
protobuf functions, which
+ # PySpark has in pyspark.sql.avro.functions and
pyspark.sql.protobuf.functions.
+ import builtins
+
+ from pyspark.sql import functions as F
+
+ assert F.__all__ == OFFICIAL_FUNCTIONS_ALL
+ namespace = {}
+ exec("from pyspark.sql.functions import *", namespace)
+ namespace.pop("__builtins__")
+ assert list(namespace) == OFFICIAL_FUNCTIONS_ALL
+ for name in ("chr", "random", "uuid", "countDistinct", "bitwiseNOT",
"toDegrees",
+ "approxCountDistinct"):
+ assert name not in namespace
+ assert callable(getattr(F, name))
+ assert F.random is F.rand
+ assert "chr" not in namespace and builtins.chr(65) == "A"
+ for name in ("from_avro", "from_avro_with_options", "to_avro",
"to_avro_with_schema",
+ "from_protobuf", "to_protobuf"):
+ assert name not in namespace
+ assert not hasattr(F, name)
+
+
+def test_avro_and_protobuf_functions_are_on_their_official_modules(tmp_path,
mock_connect):
+ import pyspark.sql.avro.functions as avro
+ import pyspark.sql.protobuf.functions as protobuf
+ from pyspark.errors import PySparkTypeError
+ from pyspark.sql import functions as F
+
+ spark, service = mock_connect
+ c = F.col("v")
+ assert type(avro.from_avro(c, "{}")).__name__ == "Column"
+ assert type(avro.from_avro("v", "{}", {"mode": "PERMISSIVE"})).__name__ ==
"Column"
+ assert type(avro.to_avro(c)).__name__ == "Column"
+ assert type(avro.to_avro(c, "{}")).__name__ == "Column"
+ with pytest.raises(PySparkTypeError) as error:
+ avro.from_avro(1, "{}")
+ assert error.value.getCondition() == "INVALID_TYPE"
+ assert error.value.getMessageParameters() == {
+ "arg_name": "data", "arg_type": "pyspark.sql.Column or str"}
+ # As PySpark's, the descriptor file is read (it was ignored before), and
the bytes reach the
+ # server as the binary of the function's descriptor set.
+ descriptor = tmp_path / "m.desc"
+ descriptor.write_bytes(b"\x00fue-e-descriptor\xff")
+ spark.range(1).select(protobuf.from_protobuf(F.col("id"), "M",
descFilePath=str(descriptor))).collect()
+ assert b"\x00fue-e-descriptor\xff" in service.execute_requests[-1]
+
+
+def
test_numpy_scalar_literals_are_typed_by_their_dtype_as_pyspark(mock_connect):
+ import struct
+
+ np = pytest.importorskip("numpy")
+ from pyspark.sql import functions as F
+
+ spark, service = mock_connect
+ # As PySpark's LiteralExpression of a NumPy scalar, typed by its dtype: a
bool, a signed int
+ # of 8 to 64 bits (tinyint to bigint), float32 (float) and float64
(double). The Literal
+ # fields are: boolean=3, byte=4, short=5, integer=6, long=7, float=10,
double=11, string=13,
+ # binary=2.
+ cases = [
+ (np.bool_(True), 3, 1),
+ (np.int8(5), 4, 5),
+ (np.int16(5), 5, 5),
+ (np.int32(5), 6, 5),
+ (np.int64(5), 7, 5),
+ (np.int64(2**40), 7, 2**40),
+ (np.float32(1.5), 10, struct.pack("<f", 1.5)),
+ (np.float64(1.5), 11, struct.pack("<d", 1.5)),
+ (np.str_("abc"), 13, b"abc"),
+ (np.bytes_(b"ab"), 2, b"ab"),
+ # A Python int and float keep their types: int (integer) and double.
+ (5, 6, 5),
+ (1.5, 11, struct.pack("<d", 1.5)),
+ ]
+ for value, field, expected in cases:
+ spark.range(1).select(F.lit(value)).collect()
+ plan =
parse_message(parse_message(service.execute_requests[-1])[3][0])[1][0]
+ expression =
parse_message(parse_message(parse_message(plan)[3][0])[3][0])
+ literal = parse_message(expression[1][0])
+ assert sorted(literal) == [field], repr(value)
+ assert literal[field] == [expected], repr(value)
+
+
+def test_numpy_scalars_of_other_types_are_unsupported_as_pyspark(mock_connect):
+ np = pytest.importorskip("numpy")
+ from pyspark.errors import PySparkTypeError
+ from pyspark.sql import functions as F
+
+ # As PySpark's, the unsupported NumPy scalars are UNSUPPORTED_DATA_TYPE,
named by their
+ # Python type; and an array of an unsupported dtype is
UNSUPPORTED_NUMPY_ARRAY_SCALAR.
+ for value, type_name in [
+ (np.uint8(5), "uint8"),
+ (np.uint64(5), "uint64"),
+ (np.float16(1.5), "float16"),
+ (np.longdouble(1.5), "longdouble"),
+ (np.complex128(1 + 2j), "complex128"),
+ (np.datetime64("2020-01-01"), "datetime64"),
+ (np.timedelta64(1, "D"), "timedelta64"),
+ ]:
+ with pytest.raises(PySparkTypeError) as error:
+ F.lit(value)
+ assert error.value.getCondition() == "UNSUPPORTED_DATA_TYPE"
+ assert error.value.getMessageParameters() == {"data_type": type_name}
+ with pytest.raises(PySparkTypeError) as error:
+ F.lit(np.array([1], dtype=np.uint8))
+ assert error.value.getCondition() == "UNSUPPORTED_NUMPY_ARRAY_SCALAR"
+ assert error.value.getMessageParameters() == {"dtype": "uint8"}
+
+
# ---------------------------------------------------------------------------
4.3.0 gaps
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]