This is an automated email from the ASF dual-hosted git repository.
yangjie01 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 0a45a8662db [SPARK-30848][SQL] Remove productHash from TreeNode
0a45a8662db is described below
commit 0a45a8662dbdeb644ea3950cdaa6e3ca328cd6fc
Author: panbingkun <[email protected]>
AuthorDate: Sat Oct 21 00:07:01 2023 +0800
[SPARK-30848][SQL] Remove productHash from TreeNode
### What changes were proposed in this pull request?
The pr aims to replace `TreeNode.productHash` with
`MurmurHash3.productHash`.
### Why are the changes needed?
1.Spark 4.0 no longer supports Scala 2.12.
2.Using MurmurHash3's class method `productHash` to reduce code redundancy.
https://github.com/scala/scala/blob/v2.13.11/src/library/scala/util/hashing/MurmurHash3.scala#L343
https://github.com/scala/scala/blob/v2.13.11/src/library/scala/util/hashing/MurmurHash3.scala#L64-L81
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
1.Pass GA.
2.Manually test:
```
(base) panbingkun:~/Developer/spark/spark-community$./build/sbt
"catalyst/testOnly org.apache.spark.sql.catalyst.expressions.CanonicalizeSuite
-- -t \"SPARK-30847: Take productPrefix into account in
MurmurHash3.productHash\""
[info] CanonicalizeSuite:
[info] - SPARK-30847: Take productPrefix into account in
MurmurHash3.productHash (249 milliseconds)
[info] Run completed in 1 second, 135 milliseconds.
[info] Total number of tests run: 1
[info] Suites: completed 1, aborted 0
[info] Tests: succeeded 1, failed 0, canceled 0, ignored 0, pending 0
[info] All tests passed.
[success] Total time: 230 s (03:50), completed Sep 28, 2023, 12:54:06 PM
```
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #43161 from panbingkun/SPARK-45366.
Authored-by: panbingkun <[email protected]>
Signed-off-by: yangjie01 <[email protected]>
---
.../apache/spark/sql/catalyst/trees/TreeNode.scala | 26 ++--------------------
1 file changed, 2 insertions(+), 24 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
index cc470d0de6f..e7be8a7e29b 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
@@ -22,6 +22,7 @@ import java.util.UUID
import scala.collection.{mutable, Map}
import scala.jdk.CollectionConverters._
import scala.reflect.ClassTag
+import scala.util.hashing.MurmurHash3
import org.apache.commons.lang3.ClassUtils
import org.json4s.JsonAST._
@@ -175,30 +176,7 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]]
lazy val height: Int = children.map(_.height).reduceOption(_ max
_).getOrElse(0) + 1
- // Copied from Scala 2.13.1
- //
github.com/scala/scala/blob/v2.13.1/src/library/scala/util/hashing/MurmurHash3.scala#L56-L73
- // to prevent the issue https://github.com/scala/bug/issues/10495
- // TODO(SPARK-30848): Remove this once we drop Scala 2.12.
- private final def productHash(x: Product, seed: Int, ignorePrefix: Boolean =
false): Int = {
- val arr = x.productArity
- // Case objects have the hashCode inlined directly into the
- // synthetic hashCode method, but this method should still give
- // a correct result if passed a case object.
- if (arr == 0) {
- x.productPrefix.hashCode
- } else {
- var h = seed
- if (!ignorePrefix) h = scala.util.hashing.MurmurHash3.mix(h,
x.productPrefix.hashCode)
- var i = 0
- while (i < arr) {
- h = scala.util.hashing.MurmurHash3.mix(h, x.productElement(i).##)
- i += 1
- }
- scala.util.hashing.MurmurHash3.finalizeHash(h, arr)
- }
- }
-
- private lazy val _hashCode: Int = productHash(this,
scala.util.hashing.MurmurHash3.productSeed)
+ private lazy val _hashCode: Int = MurmurHash3.productHash(this)
override def hashCode(): Int = _hashCode
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
