Github user ueshin commented on a diff in the pull request:
https://github.com/apache/spark/pull/21061#discussion_r194523480
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
---
@@ -2189,3 +2189,293 @@ case class ArrayRemove(left: Expression, right:
Expression)
override def prettyName: String = "array_remove"
}
+
+object ArraySetLike {
+ def useGenericArrayData(elementSize: Int, length: Int): Boolean = {
+ // Use the same calculation in UnsafeArrayData.fromPrimitiveArray()
+ val headerInBytes =
UnsafeArrayData.calculateHeaderPortionInBytes(length)
+ val valueRegionInBytes = elementSize.toLong * length
+ val totalSizeInLongs = (headerInBytes + valueRegionInBytes + 7) / 8
+ totalSizeInLongs > Integer.MAX_VALUE / 8
+ }
+
+ def throwUnionLengthOverflowException(length: Int): Unit = {
+ throw new RuntimeException(s"Unsuccessful try to union arrays with
${length}" +
+ s"elements due to exceeding the array size limit " +
+ s"${ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH}.")
+ }
+
+ def evalUnionContainsNull(
--- End diff --
We should move this to `ArrayUnion` companion object?
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
