PHILO-HE commented on code in PR #5626:
URL: https://github.com/apache/incubator-gluten/pull/5626#discussion_r1616704118
##########
gluten-core/src/main/java/org/apache/gluten/substrait/expression/WindowFunctionNode.java:
##########
@@ -80,20 +91,50 @@ private Expression.WindowFunction.Bound.Builder setBound(
builder.setUnboundedFollowing(followingBuilder.build());
break;
default:
- try {
- Long offset = Long.valueOf(boundType);
- if (offset < 0) {
- Expression.WindowFunction.Bound.Preceding.Builder
offsetPrecedingBuilder =
- Expression.WindowFunction.Bound.Preceding.newBuilder();
- offsetPrecedingBuilder.setOffset(0 - offset);
- builder.setPreceding(offsetPrecedingBuilder.build());
- } else {
- Expression.WindowFunction.Bound.Following.Builder
offsetFollowingBuilder =
- Expression.WindowFunction.Bound.Following.newBuilder();
- offsetFollowingBuilder.setOffset(offset);
- builder.setFollowing(offsetFollowingBuilder.build());
+ if (boundType instanceof PreComputeRangeFrameBound) {
+ ExpressionNode refNode =
+ ExpressionConverter.replaceWithExpressionTransformer(
+ ((PreComputeRangeFrameBound)
boundType).child().toAttribute(),
+
JavaConverters.asScalaIteratorConverter(originalInputAttributes.iterator())
+ .asScala()
+ .toSeq())
+ .doTransform(new HashMap<String, Long>());
+ try {
+ Long offset = Long.valueOf(boundType.eval(null).toString());
+ if (offset < 0) {
+ Expression.WindowFunction.Bound.Preceding.Builder
refPrecedingBuilder =
+ Expression.WindowFunction.Bound.Preceding.newBuilder();
+ refPrecedingBuilder.setRef(refNode.toProtobuf());
+ builder.setPreceding(refPrecedingBuilder.build());
+ } else {
+ Expression.WindowFunction.Bound.Following.Builder
refFollowingBuilder =
+ Expression.WindowFunction.Bound.Following.newBuilder();
+ refFollowingBuilder.setRef(refNode.toProtobuf());
+ builder.setFollowing(refFollowingBuilder.build());
+ }
+ } catch (NumberFormatException e) {
+ throw new UnsupportedOperationException(
+ "Unsupported Window Function Frame Type:" + boundType);
+ }
+ } else if (boundType.foldable()) {
Review Comment:
This if branch is just for ClickHouse backend? It would be nice to leave
some comments.
##########
gluten-core/src/main/java/org/apache/gluten/substrait/expression/WindowFunctionNode.java:
##########
@@ -80,20 +91,50 @@ private Expression.WindowFunction.Bound.Builder setBound(
builder.setUnboundedFollowing(followingBuilder.build());
break;
default:
- try {
- Long offset = Long.valueOf(boundType);
- if (offset < 0) {
- Expression.WindowFunction.Bound.Preceding.Builder
offsetPrecedingBuilder =
- Expression.WindowFunction.Bound.Preceding.newBuilder();
- offsetPrecedingBuilder.setOffset(0 - offset);
- builder.setPreceding(offsetPrecedingBuilder.build());
- } else {
- Expression.WindowFunction.Bound.Following.Builder
offsetFollowingBuilder =
- Expression.WindowFunction.Bound.Following.newBuilder();
- offsetFollowingBuilder.setOffset(offset);
- builder.setFollowing(offsetFollowingBuilder.build());
+ if (boundType instanceof PreComputeRangeFrameBound) {
+ ExpressionNode refNode =
+ ExpressionConverter.replaceWithExpressionTransformer(
+ ((PreComputeRangeFrameBound)
boundType).child().toAttribute(),
+
JavaConverters.asScalaIteratorConverter(originalInputAttributes.iterator())
+ .asScala()
+ .toSeq())
+ .doTransform(new HashMap<String, Long>());
+ try {
+ Long offset = Long.valueOf(boundType.eval(null).toString());
+ if (offset < 0) {
+ Expression.WindowFunction.Bound.Preceding.Builder
refPrecedingBuilder =
+ Expression.WindowFunction.Bound.Preceding.newBuilder();
+ refPrecedingBuilder.setRef(refNode.toProtobuf());
+ builder.setPreceding(refPrecedingBuilder.build());
+ } else {
+ Expression.WindowFunction.Bound.Following.Builder
refFollowingBuilder =
+ Expression.WindowFunction.Bound.Following.newBuilder();
+ refFollowingBuilder.setRef(refNode.toProtobuf());
+ builder.setFollowing(refFollowingBuilder.build());
+ }
+ } catch (NumberFormatException e) {
+ throw new UnsupportedOperationException(
+ "Unsupported Window Function Frame Type:" + boundType);
Review Comment:
Do we need to add statements to catch this exception? I think Spark parser
has already checked the validity of foldable bound (e.g., 100a is not valid).
The catching looks unnecessary.
##########
cpp/velox/substrait/SubstraitToVeloxPlan.cc:
##########
@@ -839,23 +840,33 @@ const core::WindowNode::Frame createWindowFrame(
VELOX_FAIL("the window type only support ROWS and RANGE, and the input
type is ", std::to_string(type));
}
- auto boundTypeConversion = [](::substrait::Expression_WindowFunction_Bound
boundType)
+ auto specifiedBound =
+ [&](bool hasOffset, int64_t offset, const ::substrait::Expression&
columnRef) -> core::TypedExprPtr {
+ if (hasOffset) {
+ return std::make_shared<core::ConstantTypedExpr>(BIGINT(),
variant(offset));
+ } else {
+ return exprConverter_->toVeloxExpr(columnRef, inputType);
+ }
+ };
+
+ auto boundTypeConversion = [&](::substrait::Expression_WindowFunction_Bound
boundType)
-> std::tuple<core::WindowNode::BoundType, core::TypedExprPtr> {
- // TODO: support non-literal expression.
if (boundType.has_current_row()) {
return std::make_tuple(core::WindowNode::BoundType::kCurrentRow,
nullptr);
} else if (boundType.has_unbounded_following()) {
return std::make_tuple(core::WindowNode::BoundType::kUnboundedFollowing,
nullptr);
} else if (boundType.has_unbounded_preceding()) {
return std::make_tuple(core::WindowNode::BoundType::kUnboundedPreceding,
nullptr);
} else if (boundType.has_following()) {
+ auto following = boundType.following();
return std::make_tuple(
core::WindowNode::BoundType::kFollowing,
- std::make_shared<core::ConstantTypedExpr>(BIGINT(),
variant(boundType.following().offset())));
+ specifiedBound(following.has_offset(), following.offset(),
following.ref()));
Review Comment:
Can `has_offset` be true for Velox backend after the java side handling?
##########
gluten-core/src/main/scala/org/apache/gluten/utils/PullOutProjectHelper.scala:
##########
@@ -151,6 +183,22 @@ trait PullOutProjectHelper {
case ae: AggregateExpression => rewriteAggregateExpression(ae,
expressionMap)
case other => other
}
- we.copy(windowFunction = newWindowFunc)
+
+ val newWindowSpec = we.windowSpec.frameSpecification match {
+ case swf: SpecifiedWindowFrame if needPreComputeRangeFrame(swf) =>
+ // This is guaranteed by Spark, but we still check it here
+ if (orderSpecs.size != 1) {
+ throw new GlutenNotSupportException(
Review Comment:
Let's use `GlutenException`. Generally `GlutenNotSupportException` is caught
in fallback handling for some limitation cases. But for this case, I think we
can directly let a runtime exception happen.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
