MaxGekk commented on a change in pull request #34568:
URL: https://github.com/apache/spark/pull/34568#discussion_r758654693
##########
File path:
sql/core/src/main/scala/org/apache/spark/sql/execution/SparkOptimizer.scala
##########
@@ -37,7 +36,8 @@ class SparkOptimizer(
override def earlyScanPushDownRules: Seq[Rule[LogicalPlan]] =
// TODO: move SchemaPruning into catalyst
- SchemaPruning :: V2ScanRelationPushDown :: V2Writes ::
PruneFileSourcePartitions :: Nil
+ SchemaPruning :: V2ScanRelationPushDown :: V1Writes :: V2Writes ::
+ PruneFileSourcePartitions:: Nil
Review comment:
It does slightly confuses that `V*Writes` are here. Look,
`earlyScanPushDownRules` is about:
" ... projection and filter pushdown to scans"
but V1Writes:
"... makes sure the v1 write requirement, e.g. requiredOrdering"
something like opposite thing - pulling from instead of pushing down.
##########
File path: sql/core/src/main/scala/org/apache/spark/sql/execution/SortExec.scala
##########
@@ -206,3 +179,43 @@ case class SortExec(
override protected def withNewChildInternal(newChild: SparkPlan): SortExec =
copy(child = newChild)
}
+object SortExec {
+ def createSorter(
+ sortOrder: Seq[SortOrder],
+ output: Seq[Attribute],
+ schema: StructType,
+ enableRadixSort: Boolean,
+ testSpillFrequency: Int = 0): UnsafeExternalRowSorter = {
+ val ordering = RowOrdering.create(sortOrder, output)
+
+ // The comparator for comparing prefix
+ val boundSortExpression = BindReferences.bindReference(sortOrder.head,
output)
+ val prefixComparator =
SortPrefixUtils.getPrefixComparator(boundSortExpression)
+
+ val canUseRadixSort = enableRadixSort && sortOrder.length == 1 &&
+ SortPrefixUtils.canSortFullyWithPrefix(boundSortExpression)
+
+ // The generator for prefix
+ val prefixExpr = SortPrefix(boundSortExpression)
+ val prefixProjection = UnsafeProjection.create(Seq(prefixExpr))
+ val prefixComputer = new UnsafeExternalRowSorter.PrefixComputer {
+ private val result = new UnsafeExternalRowSorter.PrefixComputer.Prefix
+ override def computePrefix(row: InternalRow):
+ UnsafeExternalRowSorter.PrefixComputer.Prefix = {
Review comment:
Please, fix indentation like in the original code.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
