This is an automated email from the ASF dual-hosted git repository.
zero323 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 8238cdd [SPARK-37140][PYTHON] Inline type hints for
python/pyspark/resultiterable.py
8238cdd is described below
commit 8238cdd443d5bc8557f7e0e76f40c66bf8715c5a
Author: dch nguyen <[email protected]>
AuthorDate: Fri Oct 29 17:26:20 2021 +0200
[SPARK-37140][PYTHON] Inline type hints for python/pyspark/resultiterable.py
### What changes were proposed in this pull request?
Inline type hints for python/pyspark/resultiterable.py
### Why are the changes needed?
We can take advantage of static type checking within the functions by
inlining the type hints.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Existing tests
Closes #34413 from dchvn/SPARK-37140.
Authored-by: dch nguyen <[email protected]>
Signed-off-by: zero323 <[email protected]>
---
python/pyspark/resultiterable.py | 21 +++++++++++++--------
python/pyspark/resultiterable.pyi | 30 ------------------------------
2 files changed, 13 insertions(+), 38 deletions(-)
diff --git a/python/pyspark/resultiterable.py b/python/pyspark/resultiterable.py
index cd2a595..7f35cb6 100644
--- a/python/pyspark/resultiterable.py
+++ b/python/pyspark/resultiterable.py
@@ -15,26 +15,31 @@
# limitations under the License.
#
-from collections.abc import Iterable
+from typing import TypeVar, TYPE_CHECKING, Iterator, Iterable
+
+if TYPE_CHECKING:
+ from pyspark._typing import SizedIterable
__all__ = ["ResultIterable"]
+T = TypeVar("T")
+
-class ResultIterable(Iterable):
+class ResultIterable(Iterable[T]):
"""
A special result iterable. This is used because the standard
iterator can not be pickled
"""
- def __init__(self, data):
- self.data = data
- self.index = 0
- self.maxindex = len(data)
+ def __init__(self, data: "SizedIterable[T]"):
+ self.data: "SizedIterable[T]" = data
+ self.index: int = 0
+ self.maxindex: int = len(data)
- def __iter__(self):
+ def __iter__(self) -> Iterator[T]:
return iter(self.data)
- def __len__(self):
+ def __len__(self) -> int:
return len(self.data)
diff --git a/python/pyspark/resultiterable.pyi
b/python/pyspark/resultiterable.pyi
deleted file mode 100644
index 69596ad..0000000
--- a/python/pyspark/resultiterable.pyi
+++ /dev/null
@@ -1,30 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from pyspark._typing import SizedIterable
-from typing import Iterator, TypeVar
-
-T = TypeVar("T")
-
-class ResultIterable(SizedIterable[T]):
- data: SizedIterable[T]
- index: int
- maxindex: int
- def __init__(self, data: SizedIterable[T]) -> None: ...
- def __iter__(self) -> Iterator[T]: ...
- def __len__(self) -> int: ...
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
