This is an automated email from the ASF dual-hosted git repository.
mihaibudiu pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/calcite.git
The following commit(s) were added to refs/heads/main by this push:
new 81d93213e6 [CALCITE-6242] Enhance lambda closure
81d93213e6 is described below
commit 81d93213e6d2427e4116c51981db035e29f1f697
Author: Terran <[email protected]>
AuthorDate: Fri May 8 16:31:32 2026 +0800
[CALCITE-6242] Enhance lambda closure
---
.../adapter/enumerable/RexToLixTranslator.java | 7 +-
.../java/org/apache/calcite/plan/RelOptUtil.java | 13 ++
.../org/apache/calcite/rex/RexBiVisitorImpl.java | 12 +-
.../org/apache/calcite/rex/RexProgramBuilder.java | 5 +-
.../org/apache/calcite/rex/RexVisitorImpl.java | 11 +-
.../apache/calcite/runtime/CalciteResource.java | 6 +
.../sql/validate/SqlAbstractConformance.java | 4 +
.../calcite/sql/validate/SqlConformance.java | 22 +++
.../calcite/sql/validate/SqlConformanceEnum.java | 11 ++
.../sql/validate/SqlDelegatingConformance.java | 4 +
.../calcite/sql/validate/SqlLambdaScope.java | 42 +++--
.../calcite/sql/validate/SqlValidatorImpl.java | 31 +++-
.../apache/calcite/sql2rel/SqlToRelConverter.java | 23 ++-
.../calcite/runtime/CalciteResource.properties | 2 +
.../apache/calcite/test/SqlToRelConverterTest.java | 13 ++
.../org/apache/calcite/test/SqlValidatorTest.java | 198 ++++++++++++++++++++-
.../apache/calcite/test/SqlToRelConverterTest.xml | 11 ++
core/src/test/resources/sql/lambda.iq | 43 +++++
.../apache/calcite/sql/parser/SqlParserTest.java | 5 +
19 files changed, 431 insertions(+), 32 deletions(-)
diff --git
a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java
b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java
index 15924844ca..ea4ff65127 100644
---
a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java
+++
b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java
@@ -1861,9 +1861,14 @@ private Result toInnerStorageType(Result result, Type
storageType) {
new ParameterExpression[rexLambdaRefs.size()];
for (int i = 0; i < rexLambdaRefs.size(); i++) {
final RexLambdaRef rexLambdaRef = rexLambdaRefs.get(i);
+ // Declare lambda parameters as 'final' so that Janino can capture them
+ // from nested anonymous classes (e.g., nested lambdas like x -> y -> x
+ y).
+ // Janino requires captured local variables to be explicitly final,
+ // unlike javac which supports effectively-final variables.
parameterExpressions[i] =
Expressions.parameter(
- typeFactory.getJavaClass(rexLambdaRef.getType()),
rexLambdaRef.getName());
+ Modifier.FINAL, typeFactory.getJavaClass(rexLambdaRef.getType()),
+ rexLambdaRef.getName());
}
// Generate code for lambda expression body
diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java
b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java
index 9b24cf2ad2..b615c31d85 100644
--- a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java
+++ b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java
@@ -70,6 +70,7 @@
import org.apache.calcite.rex.RexExecutorImpl;
import org.apache.calcite.rex.RexFieldAccess;
import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLambda;
import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.rex.RexLocalRef;
import org.apache.calcite.rex.RexNode;
@@ -3363,6 +3364,12 @@ private static RexShuttle pushShuttle(final Project
project) {
@Override public RexNode visitInputRef(RexInputRef ref) {
return project.getProjects().get(ref.getIndex());
}
+
+ @Override public RexNode visitLambda(RexLambda lambda) {
+ // Lambda body references are at a different scope level.
+ // Do not remap indices inside lambda body against this project.
+ return lambda;
+ }
};
}
@@ -3386,6 +3393,12 @@ private static RexShuttle pushShuttle(final Calc calc) {
@Override public RexNode visitInputRef(RexInputRef ref) {
return projects.get(ref.getIndex());
}
+
+ @Override public RexNode visitLambda(RexLambda lambda) {
+ // Lambda body references are at a different scope level.
+ // Do not remap indices inside lambda body against this calc.
+ return lambda;
+ }
};
}
diff --git a/core/src/main/java/org/apache/calcite/rex/RexBiVisitorImpl.java
b/core/src/main/java/org/apache/calcite/rex/RexBiVisitorImpl.java
index 5a11bf771e..0c8f220bff 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexBiVisitorImpl.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexBiVisitorImpl.java
@@ -119,8 +119,18 @@ protected RexBiVisitorImpl(boolean deep) {
return null;
}
+ /**
+ * Visits a lambda expression. When {@code deep} is true, recurses into
+ * the lambda body so that analysis visitors (e.g. InputFinder) can discover
+ * field references inside the lambda. When {@code deep} is false, returns
+ * null without recursing — this is the shallow traversal mode used by
+ * visitors that only need top-level information.
+ */
@Override public R visitLambda(RexLambda lambda, P arg) {
- return null;
+ if (!deep) {
+ return null;
+ }
+ return lambda.getExpression().accept(this, arg);
}
@Override public R visitNodeAndFieldIndex(RexNodeAndFieldIndex
nodeAndFieldIndex, P arg) {
diff --git a/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java
b/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java
index 2c8cdfbd9a..ebeb12aa60 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java
@@ -910,8 +910,9 @@ private abstract class RegisterShuttle extends RexShuttle {
}
@Override public RexNode visitLambda(RexLambda lambda) {
- super.visitLambda(lambda);
- return registerInternal(lambda);
+ // Lambda body references are at a different scope level.
+ // Do not validate or register lambda body indices against this
program's input.
+ return lambda;
}
}
diff --git a/core/src/main/java/org/apache/calcite/rex/RexVisitorImpl.java
b/core/src/main/java/org/apache/calcite/rex/RexVisitorImpl.java
index 6ebbe92679..d6d5931d97 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexVisitorImpl.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexVisitorImpl.java
@@ -118,8 +118,17 @@ protected RexVisitorImpl(boolean deep) {
return null;
}
+ /**
+ * Visits a lambda expression. When {@code deep} is true, recurses into
+ * the lambda body to analyze its sub-expressions (critical for InputFinder
+ * to detect field references inside lambda bodies during
pushDownJoinConditions).
+ * When {@code deep} is false, returns null without recursing.
+ */
@Override public R visitLambda(RexLambda lambda) {
- return null;
+ if (!deep) {
+ return null;
+ }
+ return lambda.getExpression().accept(this);
}
@Override public R visitLambdaRef(RexLambdaRef lambdaRef) {
diff --git a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java
b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java
index ea8e1772c6..01ea9cbda0 100644
--- a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java
+++ b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java
@@ -244,9 +244,15 @@ ExInst<SqlValidatorException>
columnNotFoundInTableDidYouMean(String a0,
ExInst<SqlValidatorException> paramNotFoundInFunctionDidYouMean(String a0,
String a1, String a2);
+ @BaseMessage("Lambda closure is not allowed in this conformance: reference
to ''{0}'' from enclosing scope")
+ ExInst<SqlValidatorException> lambdaClosureNotAllowed(String identifier);
+
@BaseMessage("Param ''{0}'' not found in lambda expression ''{1}''")
ExInst<SqlValidatorException> paramNotFoundInLambdaExpression(String a0,
String a1);
+ @BaseMessage("Duplicate lambda parameter ''{0}''")
+ ExInst<SqlValidatorException> duplicateLambdaParameter(String paramName);
+
@BaseMessage("Operand {0} must be a query")
ExInst<SqlValidatorException> needQueryOp(String a0);
diff --git
a/core/src/main/java/org/apache/calcite/sql/validate/SqlAbstractConformance.java
b/core/src/main/java/org/apache/calcite/sql/validate/SqlAbstractConformance.java
index 82f06a48d1..9ef7559c53 100644
---
a/core/src/main/java/org/apache/calcite/sql/validate/SqlAbstractConformance.java
+++
b/core/src/main/java/org/apache/calcite/sql/validate/SqlAbstractConformance.java
@@ -149,6 +149,10 @@ public abstract class SqlAbstractConformance implements
SqlConformance {
return SqlConformanceEnum.DEFAULT.allowQualifyingCommonColumn();
}
+ @Override public boolean allowLambdaClosure() {
+ return SqlConformanceEnum.DEFAULT.allowLambdaClosure();
+ }
+
@Override public boolean allowAliasUnnestItems() {
return SqlConformanceEnum.DEFAULT.allowAliasUnnestItems();
}
diff --git
a/core/src/main/java/org/apache/calcite/sql/validate/SqlConformance.java
b/core/src/main/java/org/apache/calcite/sql/validate/SqlConformance.java
index 32b4a03b90..3ccc860a0d 100644
--- a/core/src/main/java/org/apache/calcite/sql/validate/SqlConformance.java
+++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlConformance.java
@@ -611,6 +611,28 @@ default boolean isColonFieldAccessAllowed() {
*/
boolean allowQualifyingCommonColumn();
+ /**
+ * Whether to allow lambda expressions to access variables from enclosing
+ * scopes (closure semantics).
+ *
+ * <p>For example, in a higher-order function context like:
+ *
+ * <blockquote><pre>
+ * SELECT *
+ * FROM t1
+ * JOIN t2 ON EXISTS(t1.arr, x -> x = t2.v)</pre></blockquote>
+ *
+ * <p>The {@code t2.v} from the enclosing scope would be accessible inside
+ * the lambda body if closures are allowed.
+ *
+ * <p>Among the built-in conformance levels, false in
+ * {@link SqlConformanceEnum#STRICT_92},
+ * {@link SqlConformanceEnum#STRICT_99},
+ * {@link SqlConformanceEnum#STRICT_2003};
+ * true otherwise.
+ */
+ boolean allowLambdaClosure();
+
/**
* Whether {@code VALUE} is allowed as an alternative to {@code VALUES} in
* the parser.
diff --git
a/core/src/main/java/org/apache/calcite/sql/validate/SqlConformanceEnum.java
b/core/src/main/java/org/apache/calcite/sql/validate/SqlConformanceEnum.java
index 4475c4ce80..fd083560e0 100644
--- a/core/src/main/java/org/apache/calcite/sql/validate/SqlConformanceEnum.java
+++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlConformanceEnum.java
@@ -433,6 +433,17 @@ public enum SqlConformanceEnum implements SqlConformance {
}
}
+ @Override public boolean allowLambdaClosure() {
+ switch (this) {
+ case STRICT_92:
+ case STRICT_99:
+ case STRICT_2003:
+ return false;
+ default:
+ return true;
+ }
+ }
+
@Override public boolean allowAliasUnnestItems() {
switch (this) {
case BIG_QUERY:
diff --git
a/core/src/main/java/org/apache/calcite/sql/validate/SqlDelegatingConformance.java
b/core/src/main/java/org/apache/calcite/sql/validate/SqlDelegatingConformance.java
index 0d415d8aec..daac4b5c41 100644
---
a/core/src/main/java/org/apache/calcite/sql/validate/SqlDelegatingConformance.java
+++
b/core/src/main/java/org/apache/calcite/sql/validate/SqlDelegatingConformance.java
@@ -158,6 +158,10 @@ protected SqlDelegatingConformance(SqlConformance
delegate) {
return delegate.allowQualifyingCommonColumn();
}
+ @Override public boolean allowLambdaClosure() {
+ return delegate.allowLambdaClosure();
+ }
+
@Override public boolean isValueAllowed() {
return delegate.isValueAllowed();
}
diff --git
a/core/src/main/java/org/apache/calcite/sql/validate/SqlLambdaScope.java
b/core/src/main/java/org/apache/calcite/sql/validate/SqlLambdaScope.java
index 22003912d4..0db4b45711 100644
--- a/core/src/main/java/org/apache/calcite/sql/validate/SqlLambdaScope.java
+++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlLambdaScope.java
@@ -21,15 +21,12 @@
import org.apache.calcite.sql.SqlLambda;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.type.SqlTypeName;
-import org.apache.calcite.util.Litmus;
import org.checkerframework.checker.nullness.qual.Nullable;
import java.util.HashMap;
import java.util.Map;
-import static com.google.common.base.Preconditions.checkArgument;
-
import static org.apache.calcite.util.Static.RESOURCE;
/**
@@ -54,7 +51,10 @@ public SqlLambdaScope(
/** True if the identifier matches one of the parameter names. */
public boolean isParameter(SqlIdentifier id) {
- return this.parameterTypes.containsKey(id.toString());
+ final SqlNameMatcher nameMatcher = validator.catalogReader.nameMatcher();
+ final String name = id.getSimple();
+ return parameterTypes.keySet().stream()
+ .anyMatch(paramName -> nameMatcher.matches(paramName, name));
}
@Override public SqlNode getNode() {
@@ -62,21 +62,35 @@ public boolean isParameter(SqlIdentifier id) {
}
@Override public SqlQualified fullyQualify(SqlIdentifier identifier) {
- boolean found = lambdaExpr.getParameters()
- .stream()
- .anyMatch(param -> param.equalsDeep(identifier, Litmus.IGNORE));
- if (found) {
- return SqlQualified.create(this, 1, null, identifier);
- } else {
+ if (identifier.isSimple()) {
+ final SqlNameMatcher nameMatcher = validator.catalogReader.nameMatcher();
+ final String name = identifier.getSimple();
+ boolean found = lambdaExpr.getParameters()
+ .stream()
+ .anyMatch(param ->
+ nameMatcher.matches(((SqlIdentifier) param).getSimple(), name));
+ if (found) {
+ return SqlQualified.create(this, 1, null, identifier);
+ }
+ }
+ if (!validator.config().conformance().allowLambdaClosure()) {
throw validator.newValidationError(identifier,
- RESOURCE.paramNotFoundInLambdaExpression(identifier.toString(),
lambdaExpr.toString()));
+ RESOURCE.lambdaClosureNotAllowed(identifier.toString()));
}
+ return parent.fullyQualify(identifier);
}
@Override public @Nullable RelDataType resolveColumn(String columnName,
SqlNode ctx) {
- checkArgument(parameterTypes.containsKey(columnName),
- "column %s not found", columnName);
- return parameterTypes.get(columnName);
+ final SqlNameMatcher nameMatcher = validator.catalogReader.nameMatcher();
+ for (Map.Entry<String, RelDataType> entry : parameterTypes.entrySet()) {
+ if (nameMatcher.matches(entry.getKey(), columnName)) {
+ return entry.getValue();
+ }
+ }
+ // Delegate to parent scope for nested lambda closure resolution.
+ // In a nested lambda like x -> EXISTS(arr, y -> x + y), the inner lambda
+ // scope does not contain 'x', but the outer lambda scope does.
+ return parent.resolveColumn(columnName, ctx);
}
public Map<String, RelDataType> getParameterTypes() {
diff --git
a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
index 26faab77bc..2897202926 100644
--- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
+++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
@@ -3353,10 +3353,9 @@ private void registerQuery(
alias,
lambdaNamespace,
forceNullable);
- operands = call.getOperandList();
- for (int i = 0; i < operands.size(); i++) {
- registerOperandSubQueries(parentScope, call, i);
- }
+ // Register sub-queries inside the body under lambdaScope, so that
+ // nested lambdas can resolve outer lambda parameters.
+ registerOperandSubQueries(lambdaScope, call, 1);
break;
case WITH:
@@ -6642,6 +6641,30 @@ public void setOriginal(SqlNode expr, SqlNode original) {
final LambdaNamespace ns =
getNamespaceOrThrow(lambdaExpr).unwrap(LambdaNamespace.class);
+ // Check for duplicate lambda parameter names
+ final SqlNameMatcher nameMatcher = catalogReader.nameMatcher();
+ final Set<String> seen = nameMatcher.createSet();
+ for (SqlNode param : lambdaExpr.getParameters()) {
+ final String name = ((SqlIdentifier) param).getSimple();
+ if (!seen.add(name)) {
+ throw newValidationError(param,
+ RESOURCE.duplicateLambdaParameter(name));
+ }
+ // Check against enclosing lambda scopes: x -> ... x -> ...
+ SqlValidatorScope parentScope = scope.getParent();
+ while (parentScope instanceof DelegatingScope) {
+ if (parentScope instanceof SqlLambdaScope) {
+ final SqlLambdaScope parentLambda = (SqlLambdaScope) parentScope;
+ if (parentLambda.getParameterTypes().keySet().stream()
+ .anyMatch(p -> nameMatcher.matches(p, name))) {
+ throw newValidationError(param,
+ RESOURCE.duplicateLambdaParameter(name));
+ }
+ }
+ parentScope = ((DelegatingScope) parentScope).getParent();
+ }
+ }
+
deriveType(scope, lambdaExpr.getExpression());
RelDataType type = deriveTypeImpl(scope, lambdaExpr);
setValidatedNodeType(lambdaExpr, type);
diff --git
a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index ed6d52491a..240b8bd02b 100644
--- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -2480,6 +2480,13 @@ private RexNode convertLambda(Blackboard bb, SqlNode
node) {
final SqlLambdaScope scope = (SqlLambdaScope)
validator().getLambdaScope(call);
final Map<String, RexNode> nameToNodeMap = new HashMap<>();
+ // For nested lambdas, inherit the parent blackboard's nameToNodeMap so
that
+ // the inner lambda can resolve references to outer lambda parameters.
+ // e.g., in x -> EXISTS(arr, y -> x + y = 4), the inner lambda's blackboard
+ // needs access to "X" from the outer lambda's nameToNodeMap.
+ if (bb.nameToNodeMap != null) {
+ nameToNodeMap.putAll(bb.nameToNodeMap);
+ }
final List<RexLambdaRef> parameters = new
ArrayList<>(scope.getParameterTypes().size());
final Map<String, RelDataType> parameterTypes = scope.getParameterTypes();
@@ -5759,11 +5766,16 @@ void setRoot(List<RelNode> inputs) {
SqlQualified qualified) {
if (nameToNodeMap != null && qualified.prefixLength == 1) {
RexNode node = nameToNodeMap.get(qualified.identifier.names.get(0));
- if (node == null) {
+ if (node != null) {
+ return Pair.of(node, null);
+ }
+ // If the identifier is not found in nameToNodeMap and the current
scope
+ // is a lambda scope, fall through to standard scope resolution to
allow
+ // external references (e.g., t2.v in a JOIN ON lambda expression).
+ if (!(scope instanceof SqlLambdaScope)) {
throw new AssertionError("Unknown identifier '" +
qualified.identifier
+ "' encountered while expanding expression");
}
- return Pair.of(node, null);
}
final SqlNameMatcher nameMatcher =
scope.getValidator().getCatalogReader().nameMatcher();
@@ -5782,6 +5794,13 @@ void setRoot(List<RelNode> inputs) {
// preserved.
final SqlValidatorScope ancestorScope = resolve.scope;
boolean isParent = ancestorScope != scope;
+ // When in a lambda scope, external references to tables that are part
+ // of the current blackboard's inputs should be resolved locally, not
+ // as correlation variables. The lambda blackboard inherits inputs from
+ // its parent blackboard.
+ if (isParent && scope instanceof SqlLambdaScope && inputs != null) {
+ isParent = false;
+ }
if ((inputs != null) && !isParent) {
final LookupContext rels =
new LookupContext(this, inputs, systemFieldList.size());
diff --git
a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties
b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties
index 49552b4198..27e52cbce9 100644
---
a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties
+++
b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties
@@ -398,4 +398,6 @@ SelectByCannotWithGroupBy=SELECT BY cannot be used with
GROUP BY
SelectByCannotWithOrderBy=SELECT BY cannot be used with ORDER BY
DescriptorMustBeIdentifier=The argument of DESCRIPTOR must be an identifier
MeasureAliasDuplicate=Duplicate name ''{0}'' in MATCH_RECOGNIZE MEASURE alias
list
+LambdaClosureNotAllowed=Lambda closure is not allowed in this conformance:
reference to ''{0}'' from enclosing scope
+DuplicateLambdaParameter=Duplicate lambda parameter ''{0}''
# End CalciteResource.properties
diff --git
a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java
b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java
index 1016acc410..6b3255e653 100644
--- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java
+++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java
@@ -182,6 +182,19 @@ public static void checkActualAndReferenceFiles() {
.ok();
}
+ /** Test case for nested lambda: inner lambda references outer lambda
+ * parameter. Verifies that 'x' in the inner lambda is resolved from the
+ * outer lambda scope, not treated as a table column name. */
+ @Test void testNestedLambdaExpression() {
+ final String sql =
+ "select \"EXISTS\"(array(1,2,3), x -> \"EXISTS\"(array(1,2,3), y -> x
+ y = 4))";
+ fixture()
+ .withFactory(c ->
+ c.withOperatorTable(t ->
SqlValidatorTest.operatorTableFor(SqlLibrary.SPARK)))
+ .withSql(sql)
+ .ok();
+ }
+
@Test void testDotLiteralAfterRow() {
final String sql = "select row(1,2).\"EXPR$1\" from emp";
sql(sql).ok();
diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java
b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java
index 0fb4fa4055..123dbe01d0 100644
--- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java
+++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java
@@ -8607,7 +8607,10 @@ void testGroupExpressionEquivalenceParams() {
/** Test case for
* <a
href="https://issues.apache.org/jira/browse/CALCITE-3679";>[CALCITE-3679]
- * Allow lambda expressions in SQL queries</a>. */
+ * Allow lambda expressions in SQL queries</a>.
+ * <a
href="https://issues.apache.org/jira/browse/CALCITE-6242";>[CALCITE-6242]
+ * Enhance lambda closure parsing</a>.
+ * */
@Test void testHigherOrderFunction() {
final SqlValidatorFixture s = fixture()
.withOperatorTable(MockSqlOperatorTable.standard().extend());
@@ -8621,6 +8624,10 @@ void testGroupExpressionEquivalenceParams() {
.type("RecordType(INTEGER NOT NULL EXPR$0) NOT NULL");
s.withSql("select HIGHER_ORDER_FUNCTION2(1, () -> 0.1)")
.type("RecordType(INTEGER NOT NULL EXPR$0) NOT NULL");
+ s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + ^emp.deptno^)
from emp")
+ .ok();
+ s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + ^deptno^)
from emp")
+ .ok();
// test for type check
s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> ^x + 1^)")
@@ -8638,13 +8645,70 @@ void testGroupExpressionEquivalenceParams() {
.fails("Cannot apply '(?s).*HIGHER_ORDER_FUNCTION' to arguments of
type "
+ "'HIGHER_ORDER_FUNCTION\\(<INTEGER>, <FUNCTION\\(ANY, ANY,
ANY\\) -> ANY>\\)'.*");
- // test for illegal parameters
- s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + ^emp.deptno^)
from emp")
- .fails("Param 'EMP\\.DEPTNO' not found in lambda expression "
- + "'\\(`X`, `Y`\\) -> `X` \\+ 1 \\+ `EMP`\\.`DEPTNO`'");
+ }
+
+ /** Test case for lambda closure conformance checking.
+ * Tests that lambda expressions can or cannot access variables from
enclosing
+ * scopes based on the SQL conformance level.
+ * <a
href="https://issues.apache.org/jira/browse/CALCITE-6242";>[CALCITE-6242]
+ * Enhance lambda closure parsing</a>.
+ * */
+ @Test void testLambdaClosureConformance() {
+ final SqlValidatorFixture s = fixture()
+ .withOperatorTable(MockSqlOperatorTable.standard().extend());
+
+ // Lambda accessing outer scope variable (closure)
+ // In DEFAULT conformance, closure is allowed
+ s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + deptno) from
emp")
+ .withConformance(SqlConformanceEnum.DEFAULT)
+ .ok();
+
+ // In STRICT_92, closure is NOT allowed
s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + ^deptno^)
from emp")
- .fails("Param 'DEPTNO' not found in lambda expression "
- + "'\\(`X`, `Y`\\) -> `X` \\+ 1 \\+ `DEPTNO`'");
+ .withConformance(SqlConformanceEnum.STRICT_92)
+ .fails("Lambda closure is not allowed in this conformance: "
+ + "reference to 'DEPTNO' from enclosing scope");
+
+ // In STRICT_99, closure is NOT allowed
+ s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + ^deptno^)
from emp")
+ .withConformance(SqlConformanceEnum.STRICT_99)
+ .fails("Lambda closure is not allowed in this conformance: "
+ + "reference to 'DEPTNO' from enclosing scope");
+
+ // In STRICT_2003, closure is NOT allowed
+ s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + ^deptno^)
from emp")
+ .withConformance(SqlConformanceEnum.STRICT_2003)
+ .fails("Lambda closure is not allowed in this conformance: "
+ + "reference to 'DEPTNO' from enclosing scope");
+
+ // In BABEL conformance, closure is allowed
+ s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + deptno) from
emp")
+ .withConformance(SqlConformanceEnum.BABEL)
+ .ok();
+
+ // In LENIENT conformance, closure is allowed
+ s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + deptno) from
emp")
+ .withConformance(SqlConformanceEnum.LENIENT)
+ .ok();
+
+ // Lambda using only its own parameters (no closure) - should always work
+ s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1) from emp")
+ .withConformance(SqlConformanceEnum.STRICT_92)
+ .ok();
+
+ s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> y) from emp")
+ .withConformance(SqlConformanceEnum.STRICT_92)
+ .ok();
+
+ // Test with qualified column name in closure
+ s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + emp.deptno)
from emp")
+ .withConformance(SqlConformanceEnum.DEFAULT)
+ .ok();
+
+ s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + ^emp.deptno^)
from emp")
+ .withConformance(SqlConformanceEnum.STRICT_92)
+ .fails("Lambda closure is not allowed in this conformance: "
+ + "reference to 'EMP.DEPTNO' from enclosing scope");
}
/** Test case for <a
href="https://issues.apache.org/jira/browse/CALCITE-7193";>[CALCITE-7193]
@@ -8734,6 +8798,126 @@ void testGroupExpressionEquivalenceParams() {
.assertBindType(is("RecordType(INTEGER ?0)"));
}
+ /** Test case for
+ * <a
href="https://issues.apache.org/jira/browse/CALCITE-6242";>[CALCITE-6242]
+ * Enhance lambda closure parsing</a>.
+ * Tests that nested lambda expressions validate correctly: in the
+ * expression {@code x -> EXISTS(arr, y -> x + y = 4)}, the inner lambda
+ * references 'x' from the outer lambda's scope. 'x' should not be treated
+ * like a table column name. */
+ @Test void testNestedLambdaClosure() {
+ final SqlOperatorTable opTable = operatorTableFor(SqlLibrary.SPARK);
+
+ // Nested lambda: inner lambda references outer lambda parameter
+ sql("select \"EXISTS\"(array(1,2,3), x -> \"EXISTS\"(array(1,2,3), y -> x
+ y = 4))")
+ .withOperatorTable(opTable)
+ .ok();
+
+ // Nested lambda with FROM clause: outer lambda parameter 'x' should
resolve
+ // from the outer lambda scope, not as a table column
+ sql("select \"EXISTS\"(array(1,2,3), x -> \"EXISTS\"(array(1,2,3), y -> x
+ y > deptno))"
+ + " from emp")
+ .withOperatorTable(opTable)
+ .ok();
+
+ // In STRICT mode, inner lambda referencing outer lambda param is treated
+ // as closure and is rejected
+ sql("select \"EXISTS\"(array(1,2,3), x -> \"EXISTS\"(array(1,2,3), y ->
^x^ + y = 4))"
+ + " from emp")
+ .withOperatorTable(opTable)
+ .withConformance(SqlConformanceEnum.STRICT_2003)
+ .fails("Lambda closure is not allowed in this conformance: "
+ + "reference to 'X' from enclosing scope");
+ }
+
+ /** Test case for
+ * <a
href="https://issues.apache.org/jira/browse/CALCITE-6242";>[CALCITE-6242]
+ * Enhance lambda closure parsing</a>.
+ * Tests that lambda parameter names follow the same case-sensitivity
+ * rules as other identifiers, including quoting. */
+ @Test void testLambdaParameterCaseSensitivity() {
+ final SqlOperatorTable opTable = operatorTableFor(SqlLibrary.SPARK);
+
+ // Case-insensitive mode with UNCHANGED casing:
+ // parameter defined as 'x', referenced as 'X' should match
+ final SqlValidatorFixture insensitive = fixture()
+ .withCaseSensitive(false)
+ .withUnquotedCasing(Casing.UNCHANGED)
+ .withOperatorTable(opTable);
+
+ insensitive.withSql("select \"EXISTS\"(array(1,2,3), x -> x + 1 >
0)").ok();
+ insensitive.withSql("select \"EXISTS\"(array(1,2,3), x -> X + 1 >
0)").ok();
+ insensitive.withSql("select \"EXISTS\"(array(1,2,3), X -> x + 1 >
0)").ok();
+
+ // Nested lambda: inner lambda references outer parameter with different
case
+ insensitive.withSql("select \"EXISTS\"(array(1,2,3),"
+ + " x -> \"EXISTS\"(array(1,2,3), y -> X + y = 4))").ok();
+
+ // Case-sensitive mode with UNCHANGED casing:
+ // parameter defined as 'x', referenced as 'X' should NOT match
+ final SqlValidatorFixture sensitive = fixture()
+ .withCaseSensitive(true)
+ .withUnquotedCasing(Casing.UNCHANGED)
+ .withQuoting(Quoting.DOUBLE_QUOTE)
+ .withOperatorTable(opTable);
+
+ // Same case: should work
+ sensitive.withSql("select \"EXISTS\"(array(1,2,3), x -> x + 1 > 0)").ok();
+
+ // Different case: should fail in case-sensitive mode
+ sensitive.withSql("select \"EXISTS\"(array(1,2,3), x -> ^X^ + 1 > 0)")
+ .fails("Column 'X' not found in any table");
+
+ // Quoted parameter names: quoting preserves case
+ // In default config (unquotedCasing=TO_UPPER), quoted lowercase stays
lowercase
+ final SqlValidatorFixture defaultFixture = fixture()
+ .withOperatorTable(opTable);
+
+ // Unquoted parameter 'x' is converted to 'X', unquoted reference 'x' is
also 'X'
+ defaultFixture.withSql("select \"EXISTS\"(array(1,2,3), x -> x + 1 >
0)").ok();
+ }
+
+ /** Test case for
+ * <a
href="https://issues.apache.org/jira/browse/CALCITE-6242";>[CALCITE-6242]
+ * Enhance lambda closure parsing</a>.
+ * Tests that duplicate lambda parameter names are rejected, both within
+ * a single lambda and across nested lambdas (shadowing). */
+ @Test void testLambdaDuplicateParameterName() {
+ final SqlOperatorTable opTable = operatorTableFor(SqlLibrary.SPARK);
+ final SqlValidatorFixture f = fixture().withOperatorTable(opTable);
+
+ // Same parameter name used twice in one lambda
+ f.withSql("select HIGHER_ORDER_FUNCTION(1, (x, ^x^) -> x + 1)")
+ .fails("Duplicate lambda parameter 'X'");
+
+ // Same parameter name in nested lambdas (shadowing)
+ f.withSql("select \"EXISTS\"(array(1,2,3),"
+ + " x -> \"EXISTS\"(array(1,2,3), ^x^ -> x + 1 > 0))")
+ .fails("Duplicate lambda parameter 'X'");
+
+ // Different parameter names: should work
+ f.withSql("select \"EXISTS\"(array(1,2,3),"
+ + " x -> \"EXISTS\"(array(1,2,3), y -> x + y = 4))").ok();
+
+ // Case-insensitive: x and X are same parameter (shadowing detected)
+ final SqlValidatorFixture insensitive = fixture()
+ .withCaseSensitive(false)
+ .withUnquotedCasing(Casing.UNCHANGED)
+ .withOperatorTable(opTable);
+ insensitive.withSql("select \"EXISTS\"(array(1,2,3),"
+ + " x -> \"EXISTS\"(array(1,2,3), ^X^ -> X + 1 > 0))")
+ .fails("Duplicate lambda parameter 'X'");
+
+ // Case-sensitive mode: x and X are different parameters (no shadowing)
+ final SqlValidatorFixture sensitive = fixture()
+ .withCaseSensitive(true)
+ .withUnquotedCasing(Casing.UNCHANGED)
+ .withQuoting(Quoting.DOUBLE_QUOTE)
+ .withOperatorTable(opTable);
+ sensitive.withSql("select \"EXISTS\"(array(1,2,3),"
+ + " x -> \"EXISTS\"(array(1,2,3), X -> X + 1 > 0))").ok();
+ }
+
@Test void testPercentileFunctionsBigQuery() {
final SqlOperatorTable opTable = operatorTableFor(SqlLibrary.BIG_QUERY);
final String sql = "select\n"
diff --git
a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml
b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml
index bd6a2246a1..f2ab8fec37 100644
--- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml
+++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml
@@ -5963,6 +5963,17 @@ LogicalProject(D2=[$0], D3=[$1])
LogicalFilter(condition=[=($1, $0)])
LogicalProject(D4=[+($0, 4)], D5=[+($0, 5)], D6=[+($0,
6)])
LogicalTableScan(table=[[CATALOG, SALES, DEPT]])
+]]>
+ </Resource>
+ </TestCase>
+ <TestCase name="testNestedLambdaExpression">
+ <Resource name="sql">
+ <![CDATA[select "EXISTS"(array(1,2,3), x -> "EXISTS"(array(1,2,3), y ->
x + y = 4))]]>
+ </Resource>
+ <Resource name="plan">
+ <![CDATA[
+LogicalProject(EXPR$0=[EXISTS(ARRAY(1, 2, 3), (X) -> EXISTS(ARRAY(1, 2, 3),
(Y) -> =(+(X, Y), 4)))])
+ LogicalValues(tuples=[[{ 0 }]])
]]>
</Resource>
</TestCase>
diff --git a/core/src/test/resources/sql/lambda.iq
b/core/src/test/resources/sql/lambda.iq
index 207543ec77..82808ff19d 100644
--- a/core/src/test/resources/sql/lambda.iq
+++ b/core/src/test/resources/sql/lambda.iq
@@ -102,3 +102,46 @@ select "EXISTS"(array[array[1, 2], array[3, 4]], x -> x[1]
= 1);
(1 row)
!ok
+
+# [CALCITE-6242] Enhance lambda closure parsing
+select *
+ from (select array(1, 2, 3) as arr) as t1 inner join
+ (select 1 as v) as t2 on "EXISTS"(arr, x -> x = t2.v);
++-----------+---+
+| ARR | V |
++-----------+---+
+| [1, 2, 3] | 1 |
++-----------+---+
+(1 row)
+
+!ok
+
+# Nested lambda: x -> (y -> x + y).
+# The inner closure 'x' refers to the outer lambda parameter,
+# and must not be treated like a table column name.
+# For x in (1,2,3) and y in (1,2,3), x + y = 4 holds (e.g. 1+3, 2+2, 3+1).
+select "EXISTS"(array(1, 2, 3), x -> "EXISTS"(array(1, 2, 3), y -> x + y = 4));
++--------+
+| EXPR$0 |
++--------+
+| true |
++--------+
+(1 row)
+
+!ok
+
+# Nested lambda with a FROM clause whose column is also named 'x'.
+# The inner closure 'x' must resolve to the outer lambda parameter
+# (values 1,2,3), NOT to the table column 'x' (value 100).
+# If 'x' were resolved as the column, 100 + y = 4 would never hold
+# and the result would be false.
+select "EXISTS"(array(1, 2, 3), x -> "EXISTS"(array(1, 2, 3), y -> x + y = 4))
as r
+ from (select 100 as x) as t;
++------+
+| R |
++------+
+| true |
++------+
+(1 row)
+
+!ok
diff --git
a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java
b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java
index bb7566c990..e61cc66d56 100644
--- a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java
+++ b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java
@@ -9945,6 +9945,11 @@ private static Consumer<List<? extends Throwable>>
checkWarnings(
sql("select 1 || (a, b) ^->^ a + b")
.fails(errorMessage2);
+
+ // Nested lambda: inner lambda in a function call within the outer lambda
body
+ sql("select higher_order_func(x -> higher_order_func(y -> x + y, 1), 1)
from t")
+ .ok("SELECT `HIGHER_ORDER_FUNC`(`X` -> `HIGHER_ORDER_FUNC`(`Y` -> (`X`
+ `Y`), 1), 1)\n"
+ + "FROM `T`");
}
/**
