ahshahid commented on code in PR #16570:
URL: https://github.com/apache/iceberg/pull/16570#discussion_r3338363794
##########
spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkScan.java:
##########
@@ -384,4 +388,61 @@ protected long adjustSplitSize(List<? extends ScanTask>
tasks, long splitSize) {
return splitSize;
}
}
+
+ protected static String createOrderedExprString(Stream<Expression>
exprStream) {
+ return exprStream
+ .flatMap(x -> ExpressionVisitors.visit(x,
ExpressionFlattener.INSTANCE).stream())
+ .map(Spark3Util::describe)
+ .sorted()
+ .collect(Collectors.joining(", "));
+ }
+
+ private static class ExpressionFlattener
+ extends ExpressionVisitors.ExpressionVisitor<List<Expression>> {
+
+ private static final ExpressionFlattener INSTANCE = new
ExpressionFlattener();
+
+ private ExpressionFlattener() {}
+
+ @Override
+ public List<Expression> alwaysTrue() {
+ return List.of(Expressions.alwaysTrue());
+ }
+
+ @Override
+ public List<Expression> alwaysFalse() {
+ return List.of(Expressions.alwaysFalse());
+ }
+
+ @Override
+ public List<Expression> not(List<Expression> result) {
+ return List.of(Expressions.not(mergeExpressions(result)));
+ }
+
+ @Override
+ public List<Expression> and(List<Expression> leftResult, List<Expression>
rightResult) {
+ List<Expression> flattened = Lists.newArrayList(leftResult);
+ flattened.addAll(rightResult);
+ return flattened;
+ }
+
+ @Override
+ public List<Expression> or(List<Expression> leftResult, List<Expression>
rightResult) {
+ return List.of(Expressions.or(mergeExpressions(leftResult),
mergeExpressions(rightResult)));
+ }
+
+ @Override
+ public <T> List<Expression> predicate(BoundPredicate<T> pred) {
+ return List.of(pred);
+ }
+
+ @Override
+ public <T> List<Expression> predicate(UnboundPredicate<T> pred) {
+ return List.of(pred);
+ }
+
+ private Expression mergeExpressions(List<Expression> toMerge) {
+ return toMerge.stream().reduce(Expressions.alwaysTrue(),
Expressions::and);
+ }
Review Comment:
added test. once the build is complete will remove WIP
##########
spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkScan.java:
##########
@@ -384,4 +388,61 @@ protected long adjustSplitSize(List<? extends ScanTask>
tasks, long splitSize) {
return splitSize;
}
}
+
+ protected static String createOrderedExprString(Stream<Expression>
exprStream) {
+ return exprStream
+ .flatMap(x -> ExpressionVisitors.visit(x,
ExpressionFlattener.INSTANCE).stream())
+ .map(Spark3Util::describe)
+ .sorted()
+ .collect(Collectors.joining(", "));
+ }
+
+ private static class ExpressionFlattener
+ extends ExpressionVisitors.ExpressionVisitor<List<Expression>> {
+
+ private static final ExpressionFlattener INSTANCE = new
ExpressionFlattener();
+
+ private ExpressionFlattener() {}
+
+ @Override
+ public List<Expression> alwaysTrue() {
+ return List.of(Expressions.alwaysTrue());
+ }
+
+ @Override
+ public List<Expression> alwaysFalse() {
+ return List.of(Expressions.alwaysFalse());
+ }
+
+ @Override
+ public List<Expression> not(List<Expression> result) {
+ return List.of(Expressions.not(mergeExpressions(result)));
+ }
+
+ @Override
+ public List<Expression> and(List<Expression> leftResult, List<Expression>
rightResult) {
+ List<Expression> flattened = Lists.newArrayList(leftResult);
+ flattened.addAll(rightResult);
+ return flattened;
+ }
+
+ @Override
+ public List<Expression> or(List<Expression> leftResult, List<Expression>
rightResult) {
+ return List.of(Expressions.or(mergeExpressions(leftResult),
mergeExpressions(rightResult)));
+ }
+
+ @Override
+ public <T> List<Expression> predicate(BoundPredicate<T> pred) {
+ return List.of(pred);
+ }
+
+ @Override
+ public <T> List<Expression> predicate(UnboundPredicate<T> pred) {
+ return List.of(pred);
+ }
+
+ private Expression mergeExpressions(List<Expression> toMerge) {
+ return toMerge.stream().reduce(Expressions.alwaysTrue(),
Expressions::and);
+ }
Review Comment:
done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
