Github user ueshin commented on a diff in the pull request:
https://github.com/apache/spark/pull/21061#discussion_r194516045
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
---
@@ -2189,3 +2189,293 @@ case class ArrayRemove(left: Expression, right:
Expression)
override def prettyName: String = "array_remove"
}
+
+object ArraySetLike {
+ def useGenericArrayData(elementSize: Int, length: Int): Boolean = {
+ // Use the same calculation in UnsafeArrayData.fromPrimitiveArray()
+ val headerInBytes =
UnsafeArrayData.calculateHeaderPortionInBytes(length)
+ val valueRegionInBytes = elementSize.toLong * length
+ val totalSizeInLongs = (headerInBytes + valueRegionInBytes + 7) / 8
+ totalSizeInLongs > Integer.MAX_VALUE / 8
+ }
+
+ def throwUnionLengthOverflowException(length: Int): Unit = {
+ throw new RuntimeException(s"Unsuccessful try to union arrays with
${length}" +
--- End diff --
nit: `$length`, and we need a extra space after it.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
