This is an automated email from the ASF dual-hosted git repository.
gaogaotiantian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 4e530b4bfe06 [SPARK-60183][PYTHON] Improve type annotation for
pandas-on-Spark datetime callbacks
4e530b4bfe06 is described below
commit 4e530b4bfe060637b0c5238a18f15825f4d8a374
Author: Tim Hsiung <[email protected]>
AuthorDate: Thu Nov 5 16:25:46 2026 -0800
[SPARK-60183][PYTHON] Improve type annotation for pandas-on-Spark datetime
callbacks
### What changes were proposed in this pull request?
Annotate callback inputs in `python/pyspark/pandas/datetimes.py` as
`pd.Series` or `pd.DataFrame`, removing all 20 `no-untyped-def` suppressions.
Keep the five `valid-type` suppressions for runtime-generated return-schema
annotations.
### Why are the changes needed?
The callbacks receive pandas objects, so their input types can be declared
directly instead of suppressing the typed-definition check. Existing return
annotations continue to define the Spark output schemas.
### Does this PR introduce _any_ user-facing change?
No. Only internal input annotations and suppression comments change;
callback logic and output schemas are unchanged.
### How was this patch tested?
- Mypy passed on all 596 files in `python/pyspark/pandas`, independently of
the accessor typing change.
- `pyspark.pandas.tests.series.test_datetime`: 42 tests total, with 35
passing, five pre-existing skips, and two expected failures.
- Separate runtime checks for `month_name()` and `day_name()` with the
default locale matched pandas output; their existing locale-specific tests are
skipped.
- Ruff lint/format checks and `git diff --check` passed.
No new tests were added; the existing datetime suite and local runtime
checks cover these callbacks.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: pi 1.1.0 with OpenAI gpt-6-astra
Closes #59393 from bearomorphism/pandas-datetime-callback-typing.
Authored-by: Tim Hsiung <[email protected]>
Signed-off-by: Tian Gao <[email protected]>
---
python/pyspark/pandas/datetimes.py | 42 +++++++++++++++++++-------------------
1 file changed, 21 insertions(+), 21 deletions(-)
diff --git a/python/pyspark/pandas/datetimes.py
b/python/pyspark/pandas/datetimes.py
index 4a1afca1fae6..7c7ab92b2098 100644
--- a/python/pyspark/pandas/datetimes.py
+++ b/python/pyspark/pandas/datetimes.py
@@ -111,7 +111,7 @@ class DatetimeMethods:
The microseconds of the datetime.
"""
- def pandas_microsecond(s) -> ps.Series[np.int32]: # type:
ignore[no-untyped-def]
+ def pandas_microsecond(s: pd.Series) -> ps.Series[np.int32]:
return s.dt.microsecond
return self._data.pandas_on_spark.transform_batch(pandas_microsecond)
@@ -154,8 +154,8 @@ class DatetimeMethods:
return_types = [self._data.index.dtype, int, int, int]
- def pandas_isocalendar( # type: ignore[no-untyped-def]
- pdf,
+ def pandas_isocalendar(
+ pdf: pd.DataFrame,
) -> ps.DataFrame[return_types]: # type: ignore[valid-type]
# cast to int64 due to UInt32 is not supported by spark
return
pdf[pdf.columns[0]].dt.isocalendar().astype(np.int64).reset_index()
@@ -211,7 +211,7 @@ class DatetimeMethods:
dtype: int32
"""
- def pandas_dayofweek(s) -> ps.Series[np.int32]: # type:
ignore[no-untyped-def]
+ def pandas_dayofweek(s: pd.Series) -> ps.Series[np.int32]:
return s.dt.dayofweek
return self._data.pandas_on_spark.transform_batch(pandas_dayofweek)
@@ -228,7 +228,7 @@ class DatetimeMethods:
The ordinal day of the year.
"""
- def pandas_dayofyear(s) -> ps.Series[np.int32]: # type:
ignore[no-untyped-def]
+ def pandas_dayofyear(s: pd.Series) -> ps.Series[np.int32]:
return s.dt.dayofyear
return self._data.pandas_on_spark.transform_batch(pandas_dayofyear)
@@ -239,7 +239,7 @@ class DatetimeMethods:
The quarter of the date.
"""
- def pandas_quarter(s) -> ps.Series[np.int32]: # type:
ignore[no-untyped-def]
+ def pandas_quarter(s: pd.Series) -> ps.Series[np.int32]:
return s.dt.quarter
return self._data.pandas_on_spark.transform_batch(pandas_quarter)
@@ -278,7 +278,7 @@ class DatetimeMethods:
dtype: bool
"""
- def pandas_is_month_start(s) -> ps.Series[bool]: # type:
ignore[no-untyped-def]
+ def pandas_is_month_start(s: pd.Series) -> ps.Series[bool]:
return s.dt.is_month_start
return
self._data.pandas_on_spark.transform_batch(pandas_is_month_start)
@@ -317,7 +317,7 @@ class DatetimeMethods:
dtype: bool
"""
- def pandas_is_month_end(s) -> ps.Series[bool]: # type:
ignore[no-untyped-def]
+ def pandas_is_month_end(s: pd.Series) -> ps.Series[bool]:
return s.dt.is_month_end
return self._data.pandas_on_spark.transform_batch(pandas_is_month_end)
@@ -367,7 +367,7 @@ class DatetimeMethods:
Name: dates, dtype: bool
"""
- def pandas_is_quarter_start(s) -> ps.Series[bool]: # type:
ignore[no-untyped-def]
+ def pandas_is_quarter_start(s: pd.Series) -> ps.Series[bool]:
return s.dt.is_quarter_start
return
self._data.pandas_on_spark.transform_batch(pandas_is_quarter_start)
@@ -417,7 +417,7 @@ class DatetimeMethods:
Name: dates, dtype: bool
"""
- def pandas_is_quarter_end(s) -> ps.Series[bool]: # type:
ignore[no-untyped-def]
+ def pandas_is_quarter_end(s: pd.Series) -> ps.Series[bool]:
return s.dt.is_quarter_end
return
self._data.pandas_on_spark.transform_batch(pandas_is_quarter_end)
@@ -456,7 +456,7 @@ class DatetimeMethods:
dtype: bool
"""
- def pandas_is_year_start(s) -> ps.Series[bool]: # type:
ignore[no-untyped-def]
+ def pandas_is_year_start(s: pd.Series) -> ps.Series[bool]:
return s.dt.is_year_start
return self._data.pandas_on_spark.transform_batch(pandas_is_year_start)
@@ -495,7 +495,7 @@ class DatetimeMethods:
dtype: bool
"""
- def pandas_is_year_end(s) -> ps.Series[bool]: # type:
ignore[no-untyped-def]
+ def pandas_is_year_end(s: pd.Series) -> ps.Series[bool]:
return s.dt.is_year_end
return self._data.pandas_on_spark.transform_batch(pandas_is_year_end)
@@ -534,7 +534,7 @@ class DatetimeMethods:
dtype: bool
"""
- def pandas_is_leap_year(s) -> ps.Series[bool]: # type:
ignore[no-untyped-def]
+ def pandas_is_leap_year(s: pd.Series) -> ps.Series[bool]:
return s.dt.is_leap_year
return self._data.pandas_on_spark.transform_batch(pandas_is_leap_year)
@@ -545,7 +545,7 @@ class DatetimeMethods:
The number of days in the month.
"""
- def pandas_daysinmonth(s) -> ps.Series[np.int32]: # type:
ignore[no-untyped-def]
+ def pandas_daysinmonth(s: pd.Series) -> ps.Series[np.int32]:
return s.dt.daysinmonth
return self._data.pandas_on_spark.transform_batch(pandas_daysinmonth)
@@ -612,7 +612,7 @@ class DatetimeMethods:
else:
ret_dtype = self._data.dtype
- def pandas_normalize(s) -> ps.Series[ret_dtype]: # type:
ignore[no-untyped-def, valid-type]
+ def pandas_normalize(s: pd.Series) -> ps.Series[ret_dtype]: # type:
ignore[valid-type]
return s.dt.normalize()
return self._data.pandas_on_spark.transform_batch(pandas_normalize)
@@ -660,7 +660,7 @@ class DatetimeMethods:
dtype: object
"""
- def pandas_strftime(s) -> ps.Series[str]: # type:
ignore[no-untyped-def]
+ def pandas_strftime(s: pd.Series) -> ps.Series[str]:
return s.dt.strftime(date_format)
return self._data.pandas_on_spark.transform_batch(pandas_strftime)
@@ -718,7 +718,7 @@ class DatetimeMethods:
else:
ret_dtype = self._data.dtype
- def pandas_round(s) -> ps.Series[ret_dtype]: # type:
ignore[no-untyped-def, valid-type]
+ def pandas_round(s: pd.Series) -> ps.Series[ret_dtype]: # type:
ignore[valid-type]
return s.dt.round(freq, *args, **kwargs)
return self._data.pandas_on_spark.transform_batch(pandas_round)
@@ -776,7 +776,7 @@ class DatetimeMethods:
else:
ret_dtype = self._data.dtype
- def pandas_floor(s) -> ps.Series[ret_dtype]: # type:
ignore[no-untyped-def, valid-type]
+ def pandas_floor(s: pd.Series) -> ps.Series[ret_dtype]: # type:
ignore[valid-type]
return s.dt.floor(freq, *args, **kwargs)
return self._data.pandas_on_spark.transform_batch(pandas_floor)
@@ -834,7 +834,7 @@ class DatetimeMethods:
else:
ret_dtype = self._data.dtype
- def pandas_ceil(s) -> ps.Series[ret_dtype]: # type:
ignore[no-untyped-def, valid-type]
+ def pandas_ceil(s: pd.Series) -> ps.Series[ret_dtype]: # type:
ignore[valid-type]
return s.dt.ceil(freq, *args, **kwargs)
return self._data.pandas_on_spark.transform_batch(pandas_ceil)
@@ -870,7 +870,7 @@ class DatetimeMethods:
dtype: object
"""
- def pandas_month_name(s) -> ps.Series[str]: # type:
ignore[no-untyped-def]
+ def pandas_month_name(s: pd.Series) -> ps.Series[str]:
return s.dt.month_name(locale=locale)
return self._data.pandas_on_spark.transform_batch(pandas_month_name)
@@ -906,7 +906,7 @@ class DatetimeMethods:
dtype: object
"""
- def pandas_day_name(s) -> ps.Series[str]: # type:
ignore[no-untyped-def]
+ def pandas_day_name(s: pd.Series) -> ps.Series[str]:
return s.dt.day_name(locale=locale)
return self._data.pandas_on_spark.transform_batch(pandas_day_name)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]