This is an automated email from the ASF dual-hosted git repository.
gaogaotiantian pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 7e1a997a1079 [SPARK-58346][PYTHON] Remove unnecessary type:
ignore[union-attr] comments in PySpark
7e1a997a1079 is described below
commit 7e1a997a1079fd6394f2836e44de7bd330205013
Author: Spenser Sun <[email protected]>
AuthorDate: Mon Jul 27 11:47:28 2026 -0700
[SPARK-58346][PYTHON] Remove unnecessary type: ignore[union-attr] comments
in PySpark
### What changes were proposed in this pull request?
Removes 16 `# type: ignore[union-attr]` comments across 4 files in
`python/pyspark` by making the implicit non-None assumption explicit at each
access site:
- **`sql/types.py` (9):** In `_parse_datatype_json_value`, each atomic-type
branch matched its regex twice - once in the `elif` test and again to bind `m`
- then suppressed the `Optional[re.Match]` on `m.group(...)`. Converting these
to the walrus form (`elif m := PATTERN.match(json_value):`) matches once,
narrows `m` for the branch body, and drops the ignore. This also removes the
redundant second match and makes these branches consistent with the
geometry/geography branches in the s [...]
- **`errors/exceptions/captured.py` (5):** The `CapturedException`
accessors access `SparkContext._jvm.PythonErrorUtils`, where `_jvm` is
`Optional[JVMView]`. Each method already asserts `SparkContext._gateway is not
None`; adding the matching `assert SparkContext._jvm is not None` (the same
pattern already used elsewhere in the file) narrows `_jvm` and removes the
ignore.
- **`sql/connect/client/core.py` (1):** The code narrowed a local `session`
via `if session is not None:` but then re-read the Optional class attribute
`PySparkSession._instantiatedSession._jvm`. Using the narrowed local
(`session._jvm`) removes the ignore and the redundant re-read.
- **`sql/catalog.py` (1):** The code asserted `sc is not None` but the
ignore was about `sc._gateway` being Optional. Adding `assert sc._gateway is
not None` completes the guard the code had already started.
### Why are the changes needed?
These ignores suppressed mypy `[union-attr]` errors that arise only because
a value is typed Optional at the access site, even though the surrounding code
guarantees it is not None. Making the assumption explicit is clearer and
matches idioms the codebase already uses nearby. The `assert` additions do not
introduce a new failure mode - the affected code already required a live
JVM/gateway on these paths.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
`mypy` passes at full scope over `python/pyspark` (1287 source files). No
behavior change; existing tests cover the affected code paths.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Code (Opus 4.8)
Closes #57521 from Spenserrrr/nonetype-guards.
Authored-by: Spenser Sun <[email protected]>
Signed-off-by: Tian Gao <[email protected]>
(cherry picked from commit 149ba07b5606a1f9443b313d74bbee8baa717ff4)
Signed-off-by: Tian Gao <[email protected]>
---
python/pyspark/errors/exceptions/captured.py | 15 +++++++----
python/pyspark/sql/catalog.py | 3 ++-
python/pyspark/sql/connect/client/core.py | 2 +-
python/pyspark/sql/types.py | 39 ++++++++++++----------------
4 files changed, 29 insertions(+), 30 deletions(-)
diff --git a/python/pyspark/errors/exceptions/captured.py
b/python/pyspark/errors/exceptions/captured.py
index adde27bf8a5d..da755de0d6ec 100644
--- a/python/pyspark/errors/exceptions/captured.py
+++ b/python/pyspark/errors/exceptions/captured.py
@@ -102,12 +102,13 @@ class CapturedException(PySparkException):
from py4j.java_gateway import is_instance_of
assert SparkContext._gateway is not None
+ assert SparkContext._jvm is not None
gw = SparkContext._gateway
if self._origin is not None and is_instance_of(
gw, self._origin, "org.apache.spark.SparkThrowable"
):
- utils = SparkContext._jvm.PythonErrorUtils # type:
ignore[union-attr]
+ utils = SparkContext._jvm.PythonErrorUtils
return utils.getCondition(self._origin)
else:
return None
@@ -121,12 +122,13 @@ class CapturedException(PySparkException):
from py4j.java_gateway import is_instance_of
assert SparkContext._gateway is not None
+ assert SparkContext._jvm is not None
gw = SparkContext._gateway
if self._origin is not None and is_instance_of(
gw, self._origin, "org.apache.spark.SparkThrowable"
):
- utils = SparkContext._jvm.PythonErrorUtils # type:
ignore[union-attr]
+ utils = SparkContext._jvm.PythonErrorUtils
return dict(utils.getMessageParameters(self._origin))
else:
return None
@@ -136,11 +138,12 @@ class CapturedException(PySparkException):
from py4j.java_gateway import is_instance_of
assert SparkContext._gateway is not None
+ assert SparkContext._jvm is not None
gw = SparkContext._gateway
if self._origin is not None and is_instance_of(
gw, self._origin, "org.apache.spark.SparkThrowable"
):
- utils = SparkContext._jvm.PythonErrorUtils # type:
ignore[union-attr]
+ utils = SparkContext._jvm.PythonErrorUtils
return utils.getSqlState(self._origin)
else:
return None
@@ -150,12 +153,13 @@ class CapturedException(PySparkException):
from py4j.java_gateway import is_instance_of
assert SparkContext._gateway is not None
+ assert SparkContext._jvm is not None
gw = SparkContext._gateway
if self._origin is not None and is_instance_of(
gw, self._origin, "org.apache.spark.SparkThrowable"
):
- utils = SparkContext._jvm.PythonErrorUtils # type:
ignore[union-attr]
+ utils = SparkContext._jvm.PythonErrorUtils
errorClass = utils.getCondition(self._origin)
messageParameters = utils.getMessageParameters(self._origin)
@@ -172,13 +176,14 @@ class CapturedException(PySparkException):
from py4j.java_gateway import is_instance_of
assert SparkContext._gateway is not None
+ assert SparkContext._jvm is not None
gw = SparkContext._gateway
if self._origin is not None and is_instance_of(
gw, self._origin, "org.apache.spark.SparkThrowable"
):
contexts: List[BaseQueryContext] = []
- utils = SparkContext._jvm.PythonErrorUtils # type:
ignore[union-attr]
+ utils = SparkContext._jvm.PythonErrorUtils
for q in utils.getQueryContext(self._origin):
if q.contextType().toString() == "SQL":
contexts.append(SQLQueryContext(q))
diff --git a/python/pyspark/sql/catalog.py b/python/pyspark/sql/catalog.py
index e1a494fda91b..d698c458b015 100644
--- a/python/pyspark/sql/catalog.py
+++ b/python/pyspark/sql/catalog.py
@@ -252,7 +252,8 @@ class Catalog:
"""
sc = self._sc
assert sc is not None
- ju = sc._gateway.jvm.java.util # type: ignore[union-attr]
+ assert sc._gateway is not None
+ ju = sc._gateway.jvm.java.util
m = ju.HashMap()
if properties:
for k, v in properties.items():
diff --git a/python/pyspark/sql/connect/client/core.py
b/python/pyspark/sql/connect/client/core.py
index 19618f2d7968..3e56a1848104 100644
--- a/python/pyspark/sql/connect/client/core.py
+++ b/python/pyspark/sql/connect/client/core.py
@@ -514,7 +514,7 @@ class DefaultChannelBuilder(ChannelBuilder):
session = PySparkSession._instantiatedSession
if session is not None:
- jvm = PySparkSession._instantiatedSession._jvm # type:
ignore[union-attr]
+ jvm = session._jvm
return getattr(
getattr(
jvm,
diff --git a/python/pyspark/sql/types.py b/python/pyspark/sql/types.py
index 2757964bcb9c..a57d045e47b7 100644
--- a/python/pyspark/sql/types.py
+++ b/python/pyspark/sql/types.py
@@ -2428,37 +2428,30 @@ def _parse_datatype_json_value( # type: ignore[return]
collation_name = collationsMap[fieldPath]
return StringType(collation_name)
return _all_mappable_types[json_value]()
- elif _FIXED_DECIMAL.match(json_value):
- m = _FIXED_DECIMAL.match(json_value)
- return DecimalType(int(m.group(1)), int(m.group(2))) # type:
ignore[union-attr]
- elif _TIME.match(json_value):
- m = _TIME.match(json_value)
- return TimeType(int(m.group(1))) # type: ignore[union-attr]
- elif _INTERVAL_DAYTIME.match(json_value):
- m = _INTERVAL_DAYTIME.match(json_value)
+ elif m := _FIXED_DECIMAL.match(json_value):
+ return DecimalType(int(m.group(1)), int(m.group(2)))
+ elif m := _TIME.match(json_value):
+ return TimeType(int(m.group(1)))
+ elif m := _INTERVAL_DAYTIME.match(json_value):
inverted_fields = DayTimeIntervalType._inverted_fields
- first_field = inverted_fields.get(m.group(1)) # type:
ignore[union-attr]
- second_field = inverted_fields.get(m.group(3)) # type:
ignore[union-attr]
+ first_field = inverted_fields.get(m.group(1))
+ second_field = inverted_fields.get(m.group(3))
if first_field is not None and second_field is None:
return DayTimeIntervalType(first_field)
return DayTimeIntervalType(first_field, second_field)
- elif _INTERVAL_YEARMONTH.match(json_value):
- m = _INTERVAL_YEARMONTH.match(json_value)
+ elif m := _INTERVAL_YEARMONTH.match(json_value):
inverted_fields = YearMonthIntervalType._inverted_fields
- first_field = inverted_fields.get(m.group(1)) # type:
ignore[union-attr]
- second_field = inverted_fields.get(m.group(3)) # type:
ignore[union-attr]
+ first_field = inverted_fields.get(m.group(1))
+ second_field = inverted_fields.get(m.group(3))
if first_field is not None and second_field is None:
return YearMonthIntervalType(first_field)
return YearMonthIntervalType(first_field, second_field)
- elif _STRING_WITH_COLLATION.match(json_value):
- m = _STRING_WITH_COLLATION.match(json_value)
- return StringType(m.group(1)) # type: ignore[union-attr]
- elif _LENGTH_CHAR.match(json_value):
- m = _LENGTH_CHAR.match(json_value)
- return CharType(int(m.group(1))) # type: ignore[union-attr]
- elif _LENGTH_VARCHAR.match(json_value):
- m = _LENGTH_VARCHAR.match(json_value)
- return VarcharType(int(m.group(1))) # type: ignore[union-attr]
+ elif m := _STRING_WITH_COLLATION.match(json_value):
+ return StringType(m.group(1))
+ elif m := _LENGTH_CHAR.match(json_value):
+ return CharType(int(m.group(1)))
+ elif m := _LENGTH_VARCHAR.match(json_value):
+ return VarcharType(int(m.group(1)))
elif _GEOMETRY.match(json_value):
return GeometryType._from_crs(GeometryType.DEFAULT_CRS)
elif _GEOMETRY_CRS.match(json_value):
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]