Github user HyukjinKwon commented on a diff in the pull request:
https://github.com/apache/spark/pull/21021#discussion_r180311923
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
---
@@ -190,28 +160,114 @@ case class SortArray(base: Expression,
ascendingOrder: Expression)
if (o1 == null && o2 == null) {
0
} else if (o1 == null) {
- 1
+ 1 * placeNullAtEnd
} else if (o2 == null) {
- -1
+ -1 * placeNullAtEnd
} else {
-ordering.compare(o1, o2)
}
}
}
}
- override def nullSafeEval(array: Any, ascending: Any): Any = {
- val elementType = base.dataType.asInstanceOf[ArrayType].elementType
+ def sortEval(array: Any, ascending: Boolean): Any = {
+ val elementType =
arrayExpression.dataType.asInstanceOf[ArrayType].elementType
val data = array.asInstanceOf[ArrayData].toArray[AnyRef](elementType)
if (elementType != NullType) {
- java.util.Arrays.sort(data, if (ascending.asInstanceOf[Boolean]) lt
else gt)
+ java.util.Arrays.sort(data, if (ascending) lt else gt)
}
new GenericArrayData(data.asInstanceOf[Array[Any]])
}
+}
+
+/**
+ * Sorts the input array in ascending / descending order according to the
natural ordering of
+ * the array elements and returns it.
+ */
+// scalastyle:off line.size.limit
+@ExpressionDescription(
+ usage = "_FUNC_(array[, ascendingOrder]) - Sorts the input array in
ascending or descending order according to the natural ordering of the array
elements.",
+ examples = """
+ Examples:
+ > SELECT _FUNC_(array('b', 'd', 'c', 'a'), true);
+ ["a","b","c","d"]
+ """)
+// scalastyle:on line.size.limit
+case class SortArray(base: Expression, ascendingOrder: Expression)
+ extends BinaryExpression with ArraySortUtil {
+
+ def this(e: Expression) = this(e, Literal(true))
+
+ override def left: Expression = base
+ override def right: Expression = ascendingOrder
+ override def dataType: DataType = base.dataType
+ override def inputTypes: Seq[AbstractDataType] = Seq(ArrayType,
BooleanType)
+
+ override def arrayExpression: Expression = base
+
+ override def checkInputDataTypes(): TypeCheckResult = base.dataType
match {
+ case ArrayType(dt, _) if RowOrdering.isOrderable(dt) =>
+ ascendingOrder match {
+ case Literal(_: Boolean, BooleanType) =>
+ TypeCheckResult.TypeCheckSuccess
+ case _ =>
+ TypeCheckResult.TypeCheckFailure(
+ "Sort order in second argument requires a boolean literal.")
+ }
+ case ArrayType(dt, _) =>
+ TypeCheckResult.TypeCheckFailure(
+ s"$prettyName does not support sorting array of type
${dt.simpleString}")
+ case _ =>
+ TypeCheckResult.TypeCheckFailure(s"$prettyName only supports array
input.")
+ }
+
+ override def nullSafeEval(array: Any, ascending: Any): Any = {
+ sortEval(array, ascending.asInstanceOf[Boolean])
+ }
override def prettyName: String = "sort_array"
}
+/**
+ * Sorts the input array in ascending order according to the natural
ordering of
+ * the array elements and returns it.
+ */
+// scalastyle:off line.size.limit
+@ExpressionDescription(
+ usage = """
+ _FUNC_(array) - Sorts the input array in ascending order. The elements
of the input array must
+ be orderable. Null elements will be placed at the end of the
returned array.""",
+ examples = """
+ Examples:
+ > SELECT _FUNC_(array('b', 'd', null, 'c', 'a'));
+ ["a","b","c","d",null]
--- End diff --
Every expression needs it .. many of functions don't have it because that
field was added from 2.3.0. You could do it:
https://github.com/apache/spark/blob/2ca9bb083c515511d2bfee271fc3e0269aceb9d5/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala#L511
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
