Akshat-Jain commented on code in PR #16729:
URL: https://github.com/apache/druid/pull/16729#discussion_r1680755758
##########
extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/WindowOperatorQueryKit.java:
##########
@@ -201,41 +267,46 @@ private boolean ifEmptyOverPresentInWindowOperstors(
operatorFactoryList = new ArrayList<>();
} else if (of instanceof NaivePartitioningOperatorFactory) {
if (((NaivePartitioningOperatorFactory)
of).getPartitionColumns().isEmpty()) {
+ // TODO: This logic need to be revamped in the future. We probably
don't need to handle empty over() cases separately.
operatorList.clear();
operatorList.add(originalQuery.getOperators());
- return true;
+ return operatorList;
}
}
}
- return false;
+ return operatorList;
}
private ShuffleSpec findShuffleSpecForNextWindow(List<OperatorFactory>
operatorFactories, int maxWorkerCount)
{
NaivePartitioningOperatorFactory partition = null;
NaiveSortOperatorFactory sort = null;
- List<KeyColumn> keyColsOfWindow = new ArrayList<>();
for (OperatorFactory of : operatorFactories) {
if (of instanceof NaivePartitioningOperatorFactory) {
partition = (NaivePartitioningOperatorFactory) of;
} else if (of instanceof NaiveSortOperatorFactory) {
sort = (NaiveSortOperatorFactory) of;
}
}
- Map<String, ColumnWithDirection.Direction> colMap = new HashMap<>();
+
+ Map<String, ColumnWithDirection.Direction> sortColumnsMap = new
HashMap<>();
if (sort != null) {
for (ColumnWithDirection sortColumn : sort.getSortColumns()) {
- colMap.put(sortColumn.getColumn(), sortColumn.getDirection());
+ sortColumnsMap.put(sortColumn.getColumn(), sortColumn.getDirection());
}
}
- assert partition != null;
- if (partition.getPartitionColumns().isEmpty()) {
+
+ if (partition == null || partition.getPartitionColumns().isEmpty()) {
+ // If operatorFactories doesn't have any partitioning factory, then we
should keep the shuffle spec from previous stage.
+ // This indicates that we already have the data partitioned correctly,
and hence we don't need to do any shuffling.
Review Comment:
As discussed offline, I'll get to this as part of the next PR. It runs into
a NPE in this PR without it for queries that have operator list as `[sort,
partition, window1, window2]`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
