This is an automated email from the ASF dual-hosted git repository. taiyang-li pushed a commit to branch fake_add_bolt_backend in repository https://gitbox.apache.org/repos/asf/gluten.git
commit 28a3a9e733158abfefafbf53504a75ac79eb051e Author: liyang.127 <[email protected]> AuthorDate: Fri Jun 26 17:52:04 2026 +0800 [GLUTEN][CORE] Defer literal node construction for InSet to reduce memory Keep raw IN-set values and the value data type on SingularOrListNode and build the substrait LiteralNode options lazily in toProtobuf(), instead of eagerly materializing one LiteralNode per value. This avoids holding many LiteralNode objects in memory for large IN lists. - Add ExpressionBuilder.makeSingularOrListNode(value, rawValues, dataType) and a matching SingularOrListNode constructor; build options from raw values lazily. - InSetTransformer passes sorted raw hset values (deterministic) and defers LiteralNode creation to protobuf serialization. Generated-by: TraeCli openrouter-3o --- .../substrait/expression/ExpressionBuilder.java | 5 ++++ .../substrait/expression/SingularOrListNode.java | 32 ++++++++++++++++++++-- .../PredicateExpressionTransformer.scala | 16 ++++++++--- 3 files changed, 47 insertions(+), 6 deletions(-) diff --git a/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/ExpressionBuilder.java b/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/ExpressionBuilder.java index 4bdef37878..547b4dd4c4 100644 --- a/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/ExpressionBuilder.java +++ b/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/ExpressionBuilder.java @@ -251,6 +251,11 @@ public class ExpressionBuilder { return new SingularOrListNode(value, expressionNodes); } + public static SingularOrListNode makeSingularOrListNode( + ExpressionNode value, List<Object> rawValues, org.apache.spark.sql.types.DataType dataType) { + return new SingularOrListNode(value, rawValues, dataType); + } + public static WindowFunctionNode makeWindowFunction( Integer functionId, List<ExpressionNode> expressionNodes, diff --git a/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/SingularOrListNode.java b/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/SingularOrListNode.java index c55791a085..6371abd676 100644 --- a/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/SingularOrListNode.java +++ b/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/SingularOrListNode.java @@ -17,6 +17,7 @@ package org.apache.gluten.substrait.expression; import io.substrait.proto.Expression; +import org.apache.spark.sql.types.DataType; import java.io.Serializable; import java.util.ArrayList; @@ -25,19 +26,46 @@ import java.util.List; public class SingularOrListNode implements ExpressionNode, Serializable { private final ExpressionNode value; private final List<ExpressionNode> listNodes = new ArrayList<>(); + // rawValues and dataType allow delaying literal node construction until toProtobuf() + private final List<Object> rawValues; + private final DataType dataType; SingularOrListNode(ExpressionNode value, List<ExpressionNode> listNodes) { this.value = value; this.listNodes.addAll(listNodes); + this.rawValues = null; + this.dataType = null; + } + + SingularOrListNode(ExpressionNode value, List<Object> rawValues, DataType dataType) { + this.value = value; + this.rawValues = new ArrayList<>(rawValues); + this.dataType = dataType; } @Override public Expression toProtobuf() { Expression.SingularOrList.Builder builder = Expression.SingularOrList.newBuilder(); builder.setValue(value.toProtobuf()); - for (ExpressionNode expressionNode : listNodes) { - builder.addOptions(expressionNode.toProtobuf()); + List<Expression> options; + if (!listNodes.isEmpty()) { + options = new ArrayList<>(listNodes.size()); + for (ExpressionNode expressionNode : listNodes) { + options.add(expressionNode.toProtobuf()); + } + } else if (rawValues != null) { + options = new ArrayList<>(rawValues.size()); + for (Object obj : rawValues) { + // construct a temporary LiteralNode and convert to protobuf to avoid keeping + // many LiteralNode objects in memory at once. Use per-value nullability. + LiteralNode literalNode = + (LiteralNode) ExpressionBuilder.makeLiteral(obj, dataType, obj == null); + options.add(literalNode.toProtobuf()); + } + } else { + options = new ArrayList<>(); } + builder.addAllOptions(options); Expression.Builder expressionBuilder = Expression.newBuilder(); expressionBuilder.setSingularOrList(builder); return expressionBuilder.build(); diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/expression/PredicateExpressionTransformer.scala b/gluten-substrait/src/main/scala/org/apache/gluten/expression/PredicateExpressionTransformer.scala index 9f443973a9..3f74cab69a 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/expression/PredicateExpressionTransformer.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/expression/PredicateExpressionTransformer.scala @@ -40,10 +40,18 @@ case class InSetTransformer( original: InSet) extends UnaryExpressionTransformer { override def doTransform(context: SubstraitContext): ExpressionNode = { - InExpressionTransformer.toTransformer( - child.doTransform(context), - original.hset, - original.child.dataType) + val leftNode = child.doTransform(context) + val values = original.hset + val valueType = original.child.dataType + + // Keep raw literal values and defer building LiteralNode objects until toProtobuf(). + val rawValues = new java.util.ArrayList[Object]( + values.toSeq + // Sort elements for deterministic behaviours. + .sortBy(Literal(_, valueType).toString()) + .map(_.asInstanceOf[Object]) + .asJava) + ExpressionBuilder.makeSingularOrListNode(leftNode, rawValues, valueType) } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
