This is an automated email from the ASF dual-hosted git repository.
ruifengz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new f223b8da9e23 [SPARK-50818][PYTHON] Replace `has_numpy` with
`have_numpy`
f223b8da9e23 is described below
commit f223b8da9e23e4e028e145e0d4dd74eeae5d2d52
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Wed Jan 15 10:04:46 2025 +0800
[SPARK-50818][PYTHON] Replace `has_numpy` with `have_numpy`
### What changes were proposed in this pull request?
Replace `has_numpy` with `have_numpy`
### Why are the changes needed?
code clean up
### Does this PR introduce _any_ user-facing change?
no
### How was this patch tested?
ci
### Was this patch authored or co-authored using generative AI tooling?
no
Closes #49490 from zhengruifeng/py_has_numpy.
Authored-by: Ruifeng Zheng <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>
---
python/pyspark/sql/functions/builtin.py | 9 +++++----
python/pyspark/sql/plot/plotly.py | 4 ++--
python/pyspark/sql/streaming/state.py | 5 +++--
.../sql/streaming/stateful_processor_api_client.py | 6 ++++--
python/pyspark/sql/types.py | 22 ++++++++++++++++------
python/pyspark/sql/utils.py | 8 --------
6 files changed, 30 insertions(+), 24 deletions(-)
diff --git a/python/pyspark/sql/functions/builtin.py
b/python/pyspark/sql/functions/builtin.py
index 6b74c9dc2dd7..93ac6da1e14c 100644
--- a/python/pyspark/sql/functions/builtin.py
+++ b/python/pyspark/sql/functions/builtin.py
@@ -65,7 +65,6 @@ from pyspark.sql.pandas.functions import pandas_udf,
PandasUDFType # noqa: F401
from pyspark.sql.utils import (
to_str as _to_str,
- has_numpy as _has_numpy,
try_remote_functions as _try_remote_functions,
get_active_spark_context as _get_active_spark_context,
enum_to_value as _enum_to_value,
@@ -79,8 +78,6 @@ if TYPE_CHECKING:
UserDefinedFunctionLike,
)
-if _has_numpy:
- import numpy as np
# Note to developers: all of PySpark functions here take string as column
names whenever possible.
# Namely, if columns are referred as arguments, they can always be both Column
or string,
@@ -254,6 +251,8 @@ def lit(col: Any) -> Column:
| [true, false]| []| [1.5, 0.1]| [a, b, c]|
+------------------+-------+-----------------+--------------------+
"""
+ from pyspark.testing.utils import have_numpy
+
if isinstance(col, Column):
return col
elif isinstance(col, list):
@@ -262,7 +261,9 @@ def lit(col: Any) -> Column:
errorClass="COLUMN_IN_LIST", messageParameters={"func_name":
"lit"}
)
return array(*[lit(item) for item in col])
- elif _has_numpy:
+ elif have_numpy:
+ import numpy as np
+
if isinstance(col, np.generic):
dt = _from_numpy_type(col.dtype)
if dt is None:
diff --git a/python/pyspark/sql/plot/plotly.py
b/python/pyspark/sql/plot/plotly.py
index c7691f144ffa..526a36033e2f 100644
--- a/python/pyspark/sql/plot/plotly.py
+++ b/python/pyspark/sql/plot/plotly.py
@@ -151,7 +151,7 @@ def plot_box(data: "DataFrame", **kwargs: Any) -> "Figure":
def plot_kde(data: "DataFrame", **kwargs: Any) -> "Figure":
- from pyspark.sql.utils import has_numpy
+ from pyspark.testing.utils import have_numpy
from pyspark.sql.pandas.utils import require_minimum_pandas_version
require_minimum_pandas_version()
@@ -166,7 +166,7 @@ def plot_kde(data: "DataFrame", **kwargs: Any) -> "Figure":
colnames = process_column_param(kwargs.pop("column", None), data)
ind = PySparkKdePlotBase.get_ind(data.select(*colnames), kwargs.pop("ind",
None))
- if has_numpy:
+ if have_numpy:
import numpy as np
if isinstance(ind, np.ndarray):
diff --git a/python/pyspark/sql/streaming/state.py
b/python/pyspark/sql/streaming/state.py
index 0ea5590ef2e6..cd067a8413e1 100644
--- a/python/pyspark/sql/streaming/state.py
+++ b/python/pyspark/sql/streaming/state.py
@@ -19,7 +19,6 @@ import json
from typing import Tuple, Optional
from pyspark.sql.types import Row, StructType, TimestampType
-from pyspark.sql.utils import has_numpy
from pyspark.errors import PySparkTypeError, PySparkValueError,
PySparkRuntimeError
__all__ = ["GroupState", "GroupStateTimeout"]
@@ -132,6 +131,8 @@ class GroupState:
"""
Update the value of the state. The value of the state cannot be null.
"""
+ from pyspark.testing.utils import have_numpy
+
if newValue is None:
raise PySparkTypeError(
errorClass="CANNOT_BE_NONE",
@@ -139,7 +140,7 @@ class GroupState:
)
converted = []
- if has_numpy:
+ if have_numpy:
import numpy as np
# In order to convert NumPy types to Python primitive types.
diff --git a/python/pyspark/sql/streaming/stateful_processor_api_client.py
b/python/pyspark/sql/streaming/stateful_processor_api_client.py
index c25b2a2392a6..6fd56481bc61 100644
--- a/python/pyspark/sql/streaming/stateful_processor_api_client.py
+++ b/python/pyspark/sql/streaming/stateful_processor_api_client.py
@@ -28,7 +28,6 @@ from pyspark.sql.types import (
Row,
)
from pyspark.sql.pandas.types import convert_pandas_using_numpy_type
-from pyspark.sql.utils import has_numpy
from pyspark.serializers import CPickleSerializer
from pyspark.errors import PySparkRuntimeError
import uuid
@@ -414,8 +413,11 @@ class StatefulProcessorApiClient:
return self.utf8_deserializer.loads(self.sockfile)
def _serialize_to_bytes(self, schema: StructType, data: Tuple) -> bytes:
+ from pyspark.testing.utils import have_numpy
+
converted = []
- if has_numpy:
+
+ if have_numpy:
import numpy as np
# In order to convert NumPy types to Python primitive types.
diff --git a/python/pyspark/sql/types.py b/python/pyspark/sql/types.py
index b913e05e16d2..cc0285233dcc 100644
--- a/python/pyspark/sql/types.py
+++ b/python/pyspark/sql/types.py
@@ -49,7 +49,6 @@ from typing import (
from pyspark.util import is_remote_only, JVM_INT_MAX
from pyspark.serializers import CloudPickleSerializer
from pyspark.sql.utils import (
- has_numpy,
get_active_spark_context,
escape_meta_characters,
StringConcat,
@@ -65,9 +64,6 @@ from pyspark.errors import (
PySparkKeyError,
)
-if has_numpy:
- import numpy as np
-
if TYPE_CHECKING:
import numpy as np
from py4j.java_gateway import GatewayClient, JavaGateway, JavaClass
@@ -3237,7 +3233,13 @@ class DayTimeIntervalTypeConverter:
class NumpyScalarConverter:
def can_convert(self, obj: Any) -> bool:
- return has_numpy and isinstance(obj, np.generic)
+ from pyspark.testing.utils import have_numpy
+
+ if have_numpy:
+ import numpy as np
+
+ return isinstance(obj, np.generic)
+ return False
def convert(self, obj: "np.generic", gateway_client: "GatewayClient") ->
Any:
return obj.item()
@@ -3248,6 +3250,8 @@ class NumpyArrayConverter:
self, nt: "np.dtype", gateway: "JavaGateway"
) -> Optional["JavaClass"]:
"""Convert NumPy type to Py4J Java type."""
+ import numpy as np
+
if nt in [np.dtype("int8"), np.dtype("int16")]:
# Mapping int8 to gateway.jvm.byte causes
# TypeError: 'bytes' object does not support item assignment
@@ -3268,7 +3272,13 @@ class NumpyArrayConverter:
return None
def can_convert(self, obj: Any) -> bool:
- return has_numpy and isinstance(obj, np.ndarray) and obj.ndim == 1
+ from pyspark.testing.utils import have_numpy
+
+ if have_numpy:
+ import numpy as np
+
+ return isinstance(obj, np.ndarray) and obj.ndim == 1
+ return False
def convert(self, obj: "np.ndarray", gateway_client: "GatewayClient") ->
"JavaGateway":
from pyspark import SparkContext
diff --git a/python/pyspark/sql/utils.py b/python/pyspark/sql/utils.py
index fbe0b274b496..855496ff3b7c 100644
--- a/python/pyspark/sql/utils.py
+++ b/python/pyspark/sql/utils.py
@@ -62,14 +62,6 @@ if TYPE_CHECKING:
from pyspark.sql.dataframe import DataFrame
from pyspark.pandas._typing import IndexOpsLike, SeriesOrIndex
-has_numpy: bool = False
-try:
- import numpy as np # noqa: F401
-
- has_numpy = True
-except ImportError:
- pass
-
FuncT = TypeVar("FuncT", bound=Callable[..., Any])
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
