cloud-fan commented on a change in pull request #23867: [SPARK-26965][SQL]
Makes ElementAt nullability more precise
URL: https://github.com/apache/spark/pull/23867#discussion_r261498779
##########
File path:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala
##########
@@ -281,10 +266,50 @@ case class GetArrayItem(child: Expression, ordinal:
Expression)
}
/**
- * Common base class for [[GetMapValue]] and [[ElementAt]].
+ * Common trait for [[GetArrayItem]] and [[ElementAt]].
*/
+trait GetArrayItemUtil {
+
+ /** `Null` is returned for invalid ordinals. */
+ protected def computeNullabilityFromArray(child: Expression, ordinal:
Expression): Boolean = {
+ if (ordinal.foldable && !ordinal.nullable) {
+ val intOrdinal = ordinal.eval().asInstanceOf[Number].intValue()
+ child match {
+ case CreateArray(ar) if intOrdinal < ar.length =>
+ ar(intOrdinal).nullable
+ case GetArrayStructFields(CreateArray(elements), field, _, _, _)
+ if intOrdinal < elements.length =>
+ elements(intOrdinal).nullable || field.nullable
+ case _ =>
+ true
+ }
+ } else {
+ true
+ }
+ }
+}
+
+/**
+ * Common trait for [[GetMapValue]] and [[ElementAt]].
+ */
+trait GetMapValueUtil extends BinaryExpression with ImplicitCastInputTypes {
+
+ /** `Null` is returned for invalid ordinals. */
+ protected def computeNullabilityFromMap: Boolean = if (right.foldable &&
!right.nullable) {
+ val keyObj = right.eval()
+ left match {
+ case m: CreateMap if m.resolved =>
+ m.keys.zip(m.values).filter { case (k, _) => k.foldable && !k.nullable
}.find {
+ case (k, _) if k.eval() == keyObj => true
Review comment:
can we rely on `==`? What if one expression returns unsafe row and the other
returns safe row?
I know this is existing code, but after a hindsight maybe it's not worth to
linear scan the map entries and just to get the precise nullability.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
