ueshin commented on code in PR #43946:
URL: https://github.com/apache/spark/pull/43946#discussion_r1402813208
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/python/UserDefinedPythonFunction.scala:
##########
@@ -237,33 +239,32 @@ class UserDefinedPythonTableFunctionAnalyzeRunner(
// Receive whether the "with single partition" property is requested.
val withSinglePartition = dataIn.readInt() == 1
- // Receive the list of requested partitioning columns, if any.
- val partitionByColumns = ArrayBuffer.empty[Expression]
- val numPartitionByColumns = dataIn.readInt()
- for (_ <- 0 until numPartitionByColumns) {
- val columnName = PythonWorkerUtils.readUTF(dataIn)
- partitionByColumns.append(UnresolvedAttribute(columnName))
+ // Receive the list of requested partitioning expressions, if any.
+ val partitionByExpressions = ArrayBuffer.empty[Expression]
+ val numPartitionByExpressions = dataIn.readInt()
+ for (_ <- 0 until numPartitionByExpressions) {
+ val expressionSql: String = PythonWorkerUtils.readUTF(dataIn)
+ val parsed: Expression = parser.parseExpression(expressionSql)
+ partitionByExpressions.append(parsed)
}
- // Receive the list of requested ordering columns, if any.
+ // Receive the list of requested ordering expressions, if any.
val orderBy = ArrayBuffer.empty[SortOrder]
val numOrderByItems = dataIn.readInt()
for (_ <- 0 until numOrderByItems) {
- val columnName = PythonWorkerUtils.readUTF(dataIn)
+ val expressionSql: String = PythonWorkerUtils.readUTF(dataIn)
+ val parsed: Expression = parser.parseExpression(expressionSql)
val direction = if (dataIn.readInt() == 1) Ascending else Descending
val overrideNullsFirst = dataIn.readInt()
overrideNullsFirst match {
- case 0 =>
- orderBy.append(SortOrder(UnresolvedAttribute(columnName), direction))
- case 1 => orderBy.append(
- SortOrder(UnresolvedAttribute(columnName), direction, NullsFirst,
Seq.empty))
- case 2 => orderBy.append(
- SortOrder(UnresolvedAttribute(columnName), direction, NullsLast,
Seq.empty))
+ case 0 => orderBy.append(SortOrder(parsed, direction))
+ case 1 => orderBy.append(SortOrder(parsed, direction, NullsFirst,
Seq.empty))
+ case 2 => orderBy.append(SortOrder(parsed, direction, NullsLast,
Seq.empty))
}
Review Comment:
Oh, does it fail? I thought it's a valid expression. NVM, then.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
