This is an automated email from the ASF dual-hosted git repository.
yangjie01 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 30c73f6e103 [SPARK-45313][CORE] Inline `Iterators#size` and remove
`Iterators.scala`
30c73f6e103 is described below
commit 30c73f6e103851ee1f3ce012572455ab3c9d5625
Author: yangjie01 <[email protected]>
AuthorDate: Tue Sep 26 00:40:29 2023 +0800
[SPARK-45313][CORE] Inline `Iterators#size` and remove `Iterators.scala`
### What changes were proposed in this pull request?
This pr inlined the code of `Iterators#size` and remove `Iterators.scala`.
### Why are the changes needed?
https://github.com/apache/spark/pull/37353 introduced optimizations based
on Scala 2.13 for the `Utils.getIteratorSize` function, hence there exist
different versions of `Iterators.scala` for Scala 2.12 and Scala 2.13.
Currently, Apache Spark 4.0 no longer supports Scala 2.12, so the
corresponding code simplification can be performed.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Pass GitHub Actions
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #43100 from LuciferYang/SPARK-45313.
Authored-by: yangjie01 <[email protected]>
Signed-off-by: yangjie01 <[email protected]>
---
.../scala/org/apache/spark/util/Iterators.scala | 40 ----------------------
.../main/scala/org/apache/spark/util/Utils.scala | 12 ++++++-
2 files changed, 11 insertions(+), 41 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/util/Iterators.scala
b/core/src/main/scala/org/apache/spark/util/Iterators.scala
deleted file mode 100644
index 9756cf49b95..00000000000
--- a/core/src/main/scala/org/apache/spark/util/Iterators.scala
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.util
-
-private[util] object Iterators {
-
- /**
- * Counts the number of elements of an iterator.
- * This method is slower than `iterator.size` when using Scala 2.13,
- * but it can avoid overflowing problem.
- */
- def size(iterator: Iterator[_]): Long = {
- // SPARK-39928: For Scala 2.13, add check of `iterator.knownSize` refer to
- // `IterableOnceOps#size` to reduce the performance gap with
`iterator.size`.
- if (iterator.knownSize > 0) iterator.knownSize.toLong
- else {
- var count = 0L
- while (iterator.hasNext) {
- count += 1L
- iterator.next()
- }
- count
- }
- }
-}
diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala
b/core/src/main/scala/org/apache/spark/util/Utils.scala
index 149071ee1b6..b9f7eccdfe1 100644
--- a/core/src/main/scala/org/apache/spark/util/Utils.scala
+++ b/core/src/main/scala/org/apache/spark/util/Utils.scala
@@ -1785,7 +1785,17 @@ private[spark] object Utils
/**
* Counts the number of elements of an iterator.
*/
- def getIteratorSize(iterator: Iterator[_]): Long = Iterators.size(iterator)
+ def getIteratorSize(iterator: Iterator[_]): Long = {
+ if (iterator.knownSize >= 0) iterator.knownSize.toLong
+ else {
+ var count = 0L
+ while (iterator.hasNext) {
+ count += 1L
+ iterator.next()
+ }
+ count
+ }
+ }
/**
* Generate a zipWithIndex iterator, avoid index value overflowing problem
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
