diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java index 61fa2b0d716..20b696dc658 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexToLixTranslator.java @@ -1802,9 +1802,14 @@ private Result toInnerStorageType(Result result, Type storageType) { new ParameterExpression[rexLambdaRefs.size()]; for (int i = 0; i < rexLambdaRefs.size(); i++) { final RexLambdaRef rexLambdaRef = rexLambdaRefs.get(i); + // Declare lambda parameters as 'final' so that Janino can capture them + // from nested anonymous classes (e.g., nested lambdas like x -> y -> x + y). + // Janino requires captured local variables to be explicitly final, + // unlike javac which supports effectively-final variables. parameterExpressions[i] = Expressions.parameter( - typeFactory.getJavaClass(rexLambdaRef.getType()), rexLambdaRef.getName()); + Modifier.FINAL, typeFactory.getJavaClass(rexLambdaRef.getType()), + rexLambdaRef.getName()); } // Generate code for lambda expression body diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java index 9b24cf2ad20..b615c31d853 100644 --- a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java +++ b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java @@ -70,6 +70,7 @@ import org.apache.calcite.rex.RexExecutorImpl; import org.apache.calcite.rex.RexFieldAccess; import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexLambda; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexLocalRef; import org.apache.calcite.rex.RexNode; @@ -3363,6 +3364,12 @@ private static RexShuttle pushShuttle(final Project project) { @Override public RexNode visitInputRef(RexInputRef ref) { return project.getProjects().get(ref.getIndex()); } + + @Override public RexNode visitLambda(RexLambda lambda) { + // Lambda body references are at a different scope level. + // Do not remap indices inside lambda body against this project. + return lambda; + } }; } @@ -3386,6 +3393,12 @@ private static RexShuttle pushShuttle(final Calc calc) { @Override public RexNode visitInputRef(RexInputRef ref) { return projects.get(ref.getIndex()); } + + @Override public RexNode visitLambda(RexLambda lambda) { + // Lambda body references are at a different scope level. + // Do not remap indices inside lambda body against this calc. + return lambda; + } }; } diff --git a/core/src/main/java/org/apache/calcite/rex/RexBiVisitorImpl.java b/core/src/main/java/org/apache/calcite/rex/RexBiVisitorImpl.java index 5a11bf771e0..0c8f220bff2 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexBiVisitorImpl.java +++ b/core/src/main/java/org/apache/calcite/rex/RexBiVisitorImpl.java @@ -119,8 +119,18 @@ protected RexBiVisitorImpl(boolean deep) { return null; } + /** + * Visits a lambda expression. When {@code deep} is true, recurses into + * the lambda body so that analysis visitors (e.g. InputFinder) can discover + * field references inside the lambda. When {@code deep} is false, returns + * null without recursing — this is the shallow traversal mode used by + * visitors that only need top-level information. + */ @Override public R visitLambda(RexLambda lambda, P arg) { - return null; + if (!deep) { + return null; + } + return lambda.getExpression().accept(this, arg); } @Override public R visitNodeAndFieldIndex(RexNodeAndFieldIndex nodeAndFieldIndex, P arg) { diff --git a/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java b/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java index 2c8cdfbd9a3..ebeb12aa609 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java +++ b/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java @@ -910,8 +910,9 @@ private abstract class RegisterShuttle extends RexShuttle { } @Override public RexNode visitLambda(RexLambda lambda) { - super.visitLambda(lambda); - return registerInternal(lambda); + // Lambda body references are at a different scope level. + // Do not validate or register lambda body indices against this program's input. + return lambda; } } diff --git a/core/src/main/java/org/apache/calcite/rex/RexVisitorImpl.java b/core/src/main/java/org/apache/calcite/rex/RexVisitorImpl.java index 6ebbe92679f..d6d5931d976 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexVisitorImpl.java +++ b/core/src/main/java/org/apache/calcite/rex/RexVisitorImpl.java @@ -118,8 +118,17 @@ protected RexVisitorImpl(boolean deep) { return null; } + /** + * Visits a lambda expression. When {@code deep} is true, recurses into + * the lambda body to analyze its sub-expressions (critical for InputFinder + * to detect field references inside lambda bodies during pushDownJoinConditions). + * When {@code deep} is false, returns null without recursing. + */ @Override public R visitLambda(RexLambda lambda) { - return null; + if (!deep) { + return null; + } + return lambda.getExpression().accept(this); } @Override public R visitLambdaRef(RexLambdaRef lambdaRef) { diff --git a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java index e00643a5673..c4b3e421f1e 100644 --- a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java +++ b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java @@ -244,9 +244,15 @@ ExInst columnNotFoundInTableDidYouMean(String a0, ExInst paramNotFoundInFunctionDidYouMean(String a0, String a1, String a2); + @BaseMessage("Lambda closure is not allowed in this conformance: reference to ''{0}'' from enclosing scope") + ExInst lambdaClosureNotAllowed(String identifier); + @BaseMessage("Param ''{0}'' not found in lambda expression ''{1}''") ExInst paramNotFoundInLambdaExpression(String a0, String a1); + @BaseMessage("Duplicate lambda parameter ''{0}''") + ExInst duplicateLambdaParameter(String paramName); + @BaseMessage("Operand {0} must be a query") ExInst needQueryOp(String a0); diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlAbstractConformance.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlAbstractConformance.java index b19d8c071d1..ffd2c6506cc 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlAbstractConformance.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlAbstractConformance.java @@ -149,6 +149,10 @@ public abstract class SqlAbstractConformance implements SqlConformance { return SqlConformanceEnum.DEFAULT.allowQualifyingCommonColumn(); } + @Override public boolean allowLambdaClosure() { + return SqlConformanceEnum.DEFAULT.allowLambdaClosure(); + } + @Override public boolean allowAliasUnnestItems() { return SqlConformanceEnum.DEFAULT.allowAliasUnnestItems(); } diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlConformance.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlConformance.java index fe902759db1..7d205595c7b 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlConformance.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlConformance.java @@ -610,6 +610,28 @@ default boolean isColonFieldAccessAllowed() { */ boolean allowQualifyingCommonColumn(); + /** + * Whether to allow lambda expressions to access variables from enclosing + * scopes (closure semantics). + * + *

For example, in a higher-order function context like: + * + *

+   * SELECT *
+   * FROM t1
+   * JOIN t2 ON EXISTS(t1.arr, x -> x = t2.v)
+ * + *

The {@code t2.v} from the enclosing scope would be accessible inside + * the lambda body if closures are allowed. + * + *

Among the built-in conformance levels, false in + * {@link SqlConformanceEnum#STRICT_92}, + * {@link SqlConformanceEnum#STRICT_99}, + * {@link SqlConformanceEnum#STRICT_2003}; + * true otherwise. + */ + boolean allowLambdaClosure(); + /** * Whether {@code VALUE} is allowed as an alternative to {@code VALUES} in * the parser. diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlConformanceEnum.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlConformanceEnum.java index 047f05981a1..80ebaf037c3 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlConformanceEnum.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlConformanceEnum.java @@ -433,6 +433,17 @@ public enum SqlConformanceEnum implements SqlConformance { } } + @Override public boolean allowLambdaClosure() { + switch (this) { + case STRICT_92: + case STRICT_99: + case STRICT_2003: + return false; + default: + return true; + } + } + @Override public boolean allowAliasUnnestItems() { switch (this) { case PRESTO: diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlDelegatingConformance.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlDelegatingConformance.java index 25f8d7e0374..f7c3671ca64 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlDelegatingConformance.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlDelegatingConformance.java @@ -158,6 +158,10 @@ protected SqlDelegatingConformance(SqlConformance delegate) { return delegate.allowQualifyingCommonColumn(); } + @Override public boolean allowLambdaClosure() { + return delegate.allowLambdaClosure(); + } + @Override public boolean isValueAllowed() { return delegate.isValueAllowed(); } diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlLambdaScope.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlLambdaScope.java index 22003912d45..0db4b457115 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlLambdaScope.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlLambdaScope.java @@ -21,15 +21,12 @@ import org.apache.calcite.sql.SqlLambda; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.type.SqlTypeName; -import org.apache.calcite.util.Litmus; import org.checkerframework.checker.nullness.qual.Nullable; import java.util.HashMap; import java.util.Map; -import static com.google.common.base.Preconditions.checkArgument; - import static org.apache.calcite.util.Static.RESOURCE; /** @@ -54,7 +51,10 @@ public SqlLambdaScope( /** True if the identifier matches one of the parameter names. */ public boolean isParameter(SqlIdentifier id) { - return this.parameterTypes.containsKey(id.toString()); + final SqlNameMatcher nameMatcher = validator.catalogReader.nameMatcher(); + final String name = id.getSimple(); + return parameterTypes.keySet().stream() + .anyMatch(paramName -> nameMatcher.matches(paramName, name)); } @Override public SqlNode getNode() { @@ -62,21 +62,35 @@ public boolean isParameter(SqlIdentifier id) { } @Override public SqlQualified fullyQualify(SqlIdentifier identifier) { - boolean found = lambdaExpr.getParameters() - .stream() - .anyMatch(param -> param.equalsDeep(identifier, Litmus.IGNORE)); - if (found) { - return SqlQualified.create(this, 1, null, identifier); - } else { + if (identifier.isSimple()) { + final SqlNameMatcher nameMatcher = validator.catalogReader.nameMatcher(); + final String name = identifier.getSimple(); + boolean found = lambdaExpr.getParameters() + .stream() + .anyMatch(param -> + nameMatcher.matches(((SqlIdentifier) param).getSimple(), name)); + if (found) { + return SqlQualified.create(this, 1, null, identifier); + } + } + if (!validator.config().conformance().allowLambdaClosure()) { throw validator.newValidationError(identifier, - RESOURCE.paramNotFoundInLambdaExpression(identifier.toString(), lambdaExpr.toString())); + RESOURCE.lambdaClosureNotAllowed(identifier.toString())); } + return parent.fullyQualify(identifier); } @Override public @Nullable RelDataType resolveColumn(String columnName, SqlNode ctx) { - checkArgument(parameterTypes.containsKey(columnName), - "column %s not found", columnName); - return parameterTypes.get(columnName); + final SqlNameMatcher nameMatcher = validator.catalogReader.nameMatcher(); + for (Map.Entry entry : parameterTypes.entrySet()) { + if (nameMatcher.matches(entry.getKey(), columnName)) { + return entry.getValue(); + } + } + // Delegate to parent scope for nested lambda closure resolution. + // In a nested lambda like x -> EXISTS(arr, y -> x + y), the inner lambda + // scope does not contain 'x', but the outer lambda scope does. + return parent.resolveColumn(columnName, ctx); } public Map getParameterTypes() { diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index e21d62aecbf..95dd6171c1e 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -3300,10 +3300,9 @@ private void registerQuery( alias, lambdaNamespace, forceNullable); - operands = call.getOperandList(); - for (int i = 0; i < operands.size(); i++) { - registerOperandSubQueries(parentScope, call, i); - } + // Register sub-queries inside the body under lambdaScope, so that + // nested lambdas can resolve outer lambda parameters. + registerOperandSubQueries(lambdaScope, call, 1); break; case WITH: @@ -6426,6 +6425,30 @@ public void setOriginal(SqlNode expr, SqlNode original) { final LambdaNamespace ns = getNamespaceOrThrow(lambdaExpr).unwrap(LambdaNamespace.class); + // Check for duplicate lambda parameter names + final SqlNameMatcher nameMatcher = catalogReader.nameMatcher(); + final Set seen = nameMatcher.createSet(); + for (SqlNode param : lambdaExpr.getParameters()) { + final String name = ((SqlIdentifier) param).getSimple(); + if (!seen.add(name)) { + throw newValidationError(param, + RESOURCE.duplicateLambdaParameter(name)); + } + // Check against enclosing lambda scopes: x -> ... x -> ... + SqlValidatorScope parentScope = scope.getParent(); + while (parentScope instanceof DelegatingScope) { + if (parentScope instanceof SqlLambdaScope) { + final SqlLambdaScope parentLambda = (SqlLambdaScope) parentScope; + if (parentLambda.getParameterTypes().keySet().stream() + .anyMatch(p -> nameMatcher.matches(p, name))) { + throw newValidationError(param, + RESOURCE.duplicateLambdaParameter(name)); + } + } + parentScope = ((DelegatingScope) parentScope).getParent(); + } + } + deriveType(scope, lambdaExpr.getExpression()); RelDataType type = deriveTypeImpl(scope, lambdaExpr); setValidatedNodeType(lambdaExpr, type); diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index 4622176a073..562195e358d 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -2362,6 +2362,13 @@ private RexNode convertLambda(Blackboard bb, SqlNode node) { final SqlLambdaScope scope = (SqlLambdaScope) validator().getLambdaScope(call); final Map nameToNodeMap = new HashMap<>(); + // For nested lambdas, inherit the parent blackboard's nameToNodeMap so that + // the inner lambda can resolve references to outer lambda parameters. + // e.g., in x -> EXISTS(arr, y -> x + y = 4), the inner lambda's blackboard + // needs access to "X" from the outer lambda's nameToNodeMap. + if (bb.nameToNodeMap != null) { + nameToNodeMap.putAll(bb.nameToNodeMap); + } final List parameters = new ArrayList<>(scope.getParameterTypes().size()); final Map parameterTypes = scope.getParameterTypes(); @@ -5563,11 +5570,16 @@ void setRoot(List inputs) { SqlQualified qualified) { if (nameToNodeMap != null && qualified.prefixLength == 1) { RexNode node = nameToNodeMap.get(qualified.identifier.names.get(0)); - if (node == null) { + if (node != null) { + return Pair.of(node, null); + } + // If the identifier is not found in nameToNodeMap and the current scope + // is a lambda scope, fall through to standard scope resolution to allow + // external references (e.g., t2.v in a JOIN ON lambda expression). + if (!(scope instanceof SqlLambdaScope)) { throw new AssertionError("Unknown identifier '" + qualified.identifier + "' encountered while expanding expression"); } - return Pair.of(node, null); } final SqlNameMatcher nameMatcher = scope.getValidator().getCatalogReader().nameMatcher(); @@ -5586,6 +5598,13 @@ void setRoot(List inputs) { // preserved. final SqlValidatorScope ancestorScope = resolve.scope; boolean isParent = ancestorScope != scope; + // When in a lambda scope, external references to tables that are part + // of the current blackboard's inputs should be resolved locally, not + // as correlation variables. The lambda blackboard inherits inputs from + // its parent blackboard. + if (isParent && scope instanceof SqlLambdaScope && inputs != null) { + isParent = false; + } if ((inputs != null) && !isParent) { final LookupContext rels = new LookupContext(this, inputs, systemFieldList.size()); diff --git a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties index 49a63bc4efd..ff89142dd8d 100644 --- a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties +++ b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties @@ -390,4 +390,6 @@ CannotInferReturnType=Cannot infer return type for {0}; operand types: {1} SelectByCannotWithGroupBy=SELECT BY cannot be used with GROUP BY SelectByCannotWithOrderBy=SELECT BY cannot be used with ORDER BY DescriptorMustBeIdentifier=The argument of DESCRIPTOR must be an identifier +LambdaClosureNotAllowed=Lambda closure is not allowed in this conformance: reference to ''{0}'' from enclosing scope +DuplicateLambdaParameter=Duplicate lambda parameter ''{0}'' # End CalciteResource.properties diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index 4bc62e7c3d5..384a68bc2bf 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -180,6 +180,19 @@ public static void checkActualAndReferenceFiles() { .ok(); } + /** Test case for nested lambda: inner lambda references outer lambda + * parameter. Verifies that 'x' in the inner lambda is resolved from the + * outer lambda scope, not treated as a table column name. */ + @Test void testNestedLambdaExpression() { + final String sql = + "select \"EXISTS\"(array(1,2,3), x -> \"EXISTS\"(array(1,2,3), y -> x + y = 4))"; + fixture() + .withFactory(c -> + c.withOperatorTable(t -> SqlValidatorTest.operatorTableFor(SqlLibrary.SPARK))) + .withSql(sql) + .ok(); + } + @Test void testDotLiteralAfterRow() { final String sql = "select row(1,2).\"EXPR$1\" from emp"; sql(sql).ok(); diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 7c08ef3925a..817d95b9914 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -8182,7 +8182,10 @@ void testGroupExpressionEquivalenceParams() { /** Test case for * [CALCITE-3679] - * Allow lambda expressions in SQL queries. */ + * Allow lambda expressions in SQL queries. + * [CALCITE-6242] + * Enhance lambda closure parsing. + * */ @Test void testHigherOrderFunction() { final SqlValidatorFixture s = fixture() .withOperatorTable(MockSqlOperatorTable.standard().extend()); @@ -8196,6 +8199,10 @@ void testGroupExpressionEquivalenceParams() { .type("RecordType(INTEGER NOT NULL EXPR$0) NOT NULL"); s.withSql("select HIGHER_ORDER_FUNCTION2(1, () -> 0.1)") .type("RecordType(INTEGER NOT NULL EXPR$0) NOT NULL"); + s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + ^emp.deptno^) from emp") + .ok(); + s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + ^deptno^) from emp") + .ok(); // test for type check s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> ^x + 1^)") @@ -8213,13 +8220,70 @@ void testGroupExpressionEquivalenceParams() { .fails("Cannot apply '(?s).*HIGHER_ORDER_FUNCTION' to arguments of type " + "'HIGHER_ORDER_FUNCTION\\(, ANY>\\)'.*"); - // test for illegal parameters - s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + ^emp.deptno^) from emp") - .fails("Param 'EMP\\.DEPTNO' not found in lambda expression " - + "'\\(`X`, `Y`\\) -> `X` \\+ 1 \\+ `EMP`\\.`DEPTNO`'"); + } + + /** Test case for lambda closure conformance checking. + * Tests that lambda expressions can or cannot access variables from enclosing + * scopes based on the SQL conformance level. + * [CALCITE-6242] + * Enhance lambda closure parsing. + * */ + @Test void testLambdaClosureConformance() { + final SqlValidatorFixture s = fixture() + .withOperatorTable(MockSqlOperatorTable.standard().extend()); + + // Lambda accessing outer scope variable (closure) + // In DEFAULT conformance, closure is allowed + s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + deptno) from emp") + .withConformance(SqlConformanceEnum.DEFAULT) + .ok(); + + // In STRICT_92, closure is NOT allowed s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + ^deptno^) from emp") - .fails("Param 'DEPTNO' not found in lambda expression " - + "'\\(`X`, `Y`\\) -> `X` \\+ 1 \\+ `DEPTNO`'"); + .withConformance(SqlConformanceEnum.STRICT_92) + .fails("Lambda closure is not allowed in this conformance: " + + "reference to 'DEPTNO' from enclosing scope"); + + // In STRICT_99, closure is NOT allowed + s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + ^deptno^) from emp") + .withConformance(SqlConformanceEnum.STRICT_99) + .fails("Lambda closure is not allowed in this conformance: " + + "reference to 'DEPTNO' from enclosing scope"); + + // In STRICT_2003, closure is NOT allowed + s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + ^deptno^) from emp") + .withConformance(SqlConformanceEnum.STRICT_2003) + .fails("Lambda closure is not allowed in this conformance: " + + "reference to 'DEPTNO' from enclosing scope"); + + // In BABEL conformance, closure is allowed + s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + deptno) from emp") + .withConformance(SqlConformanceEnum.BABEL) + .ok(); + + // In LENIENT conformance, closure is allowed + s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + deptno) from emp") + .withConformance(SqlConformanceEnum.LENIENT) + .ok(); + + // Lambda using only its own parameters (no closure) - should always work + s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1) from emp") + .withConformance(SqlConformanceEnum.STRICT_92) + .ok(); + + s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> y) from emp") + .withConformance(SqlConformanceEnum.STRICT_92) + .ok(); + + // Test with qualified column name in closure + s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + emp.deptno) from emp") + .withConformance(SqlConformanceEnum.DEFAULT) + .ok(); + + s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + ^emp.deptno^) from emp") + .withConformance(SqlConformanceEnum.STRICT_92) + .fails("Lambda closure is not allowed in this conformance: " + + "reference to 'EMP.DEPTNO' from enclosing scope"); } /** Test case for [CALCITE-7193] @@ -8309,6 +8373,126 @@ void testGroupExpressionEquivalenceParams() { .assertBindType(is("RecordType(INTEGER ?0)")); } + /** Test case for + * [CALCITE-6242] + * Enhance lambda closure parsing. + * Tests that nested lambda expressions validate correctly: in the + * expression {@code x -> EXISTS(arr, y -> x + y = 4)}, the inner lambda + * references 'x' from the outer lambda's scope. 'x' should not be treated + * like a table column name. */ + @Test void testNestedLambdaClosure() { + final SqlOperatorTable opTable = operatorTableFor(SqlLibrary.SPARK); + + // Nested lambda: inner lambda references outer lambda parameter + sql("select \"EXISTS\"(array(1,2,3), x -> \"EXISTS\"(array(1,2,3), y -> x + y = 4))") + .withOperatorTable(opTable) + .ok(); + + // Nested lambda with FROM clause: outer lambda parameter 'x' should resolve + // from the outer lambda scope, not as a table column + sql("select \"EXISTS\"(array(1,2,3), x -> \"EXISTS\"(array(1,2,3), y -> x + y > deptno))" + + " from emp") + .withOperatorTable(opTable) + .ok(); + + // In STRICT mode, inner lambda referencing outer lambda param is treated + // as closure and is rejected + sql("select \"EXISTS\"(array(1,2,3), x -> \"EXISTS\"(array(1,2,3), y -> ^x^ + y = 4))" + + " from emp") + .withOperatorTable(opTable) + .withConformance(SqlConformanceEnum.STRICT_2003) + .fails("Lambda closure is not allowed in this conformance: " + + "reference to 'X' from enclosing scope"); + } + + /** Test case for + * [CALCITE-6242] + * Enhance lambda closure parsing. + * Tests that lambda parameter names follow the same case-sensitivity + * rules as other identifiers, including quoting. */ + @Test void testLambdaParameterCaseSensitivity() { + final SqlOperatorTable opTable = operatorTableFor(SqlLibrary.SPARK); + + // Case-insensitive mode with UNCHANGED casing: + // parameter defined as 'x', referenced as 'X' should match + final SqlValidatorFixture insensitive = fixture() + .withCaseSensitive(false) + .withUnquotedCasing(Casing.UNCHANGED) + .withOperatorTable(opTable); + + insensitive.withSql("select \"EXISTS\"(array(1,2,3), x -> x + 1 > 0)").ok(); + insensitive.withSql("select \"EXISTS\"(array(1,2,3), x -> X + 1 > 0)").ok(); + insensitive.withSql("select \"EXISTS\"(array(1,2,3), X -> x + 1 > 0)").ok(); + + // Nested lambda: inner lambda references outer parameter with different case + insensitive.withSql("select \"EXISTS\"(array(1,2,3)," + + " x -> \"EXISTS\"(array(1,2,3), y -> X + y = 4))").ok(); + + // Case-sensitive mode with UNCHANGED casing: + // parameter defined as 'x', referenced as 'X' should NOT match + final SqlValidatorFixture sensitive = fixture() + .withCaseSensitive(true) + .withUnquotedCasing(Casing.UNCHANGED) + .withQuoting(Quoting.DOUBLE_QUOTE) + .withOperatorTable(opTable); + + // Same case: should work + sensitive.withSql("select \"EXISTS\"(array(1,2,3), x -> x + 1 > 0)").ok(); + + // Different case: should fail in case-sensitive mode + sensitive.withSql("select \"EXISTS\"(array(1,2,3), x -> ^X^ + 1 > 0)") + .fails("Column 'X' not found in any table"); + + // Quoted parameter names: quoting preserves case + // In default config (unquotedCasing=TO_UPPER), quoted lowercase stays lowercase + final SqlValidatorFixture defaultFixture = fixture() + .withOperatorTable(opTable); + + // Unquoted parameter 'x' is converted to 'X', unquoted reference 'x' is also 'X' + defaultFixture.withSql("select \"EXISTS\"(array(1,2,3), x -> x + 1 > 0)").ok(); + } + + /** Test case for + * [CALCITE-6242] + * Enhance lambda closure parsing. + * Tests that duplicate lambda parameter names are rejected, both within + * a single lambda and across nested lambdas (shadowing). */ + @Test void testLambdaDuplicateParameterName() { + final SqlOperatorTable opTable = operatorTableFor(SqlLibrary.SPARK); + final SqlValidatorFixture f = fixture().withOperatorTable(opTable); + + // Same parameter name used twice in one lambda + f.withSql("select HIGHER_ORDER_FUNCTION(1, (x, ^x^) -> x + 1)") + .fails("Duplicate lambda parameter 'X'"); + + // Same parameter name in nested lambdas (shadowing) + f.withSql("select \"EXISTS\"(array(1,2,3)," + + " x -> \"EXISTS\"(array(1,2,3), ^x^ -> x + 1 > 0))") + .fails("Duplicate lambda parameter 'X'"); + + // Different parameter names: should work + f.withSql("select \"EXISTS\"(array(1,2,3)," + + " x -> \"EXISTS\"(array(1,2,3), y -> x + y = 4))").ok(); + + // Case-insensitive: x and X are same parameter (shadowing detected) + final SqlValidatorFixture insensitive = fixture() + .withCaseSensitive(false) + .withUnquotedCasing(Casing.UNCHANGED) + .withOperatorTable(opTable); + insensitive.withSql("select \"EXISTS\"(array(1,2,3)," + + " x -> \"EXISTS\"(array(1,2,3), ^X^ -> X + 1 > 0))") + .fails("Duplicate lambda parameter 'X'"); + + // Case-sensitive mode: x and X are different parameters (no shadowing) + final SqlValidatorFixture sensitive = fixture() + .withCaseSensitive(true) + .withUnquotedCasing(Casing.UNCHANGED) + .withQuoting(Quoting.DOUBLE_QUOTE) + .withOperatorTable(opTable); + sensitive.withSql("select \"EXISTS\"(array(1,2,3)," + + " x -> \"EXISTS\"(array(1,2,3), X -> X + 1 > 0))").ok(); + } + @Test void testPercentileFunctionsBigQuery() { final SqlOperatorTable opTable = operatorTableFor(SqlLibrary.BIG_QUERY); final String sql = "select\n" diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index 53662875e0d..eab455652a3 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -5676,6 +5676,17 @@ LogicalProject(D2=[$0], D3=[$1]) LogicalFilter(condition=[=($1, $0)]) LogicalProject(D4=[+($0, 4)], D5=[+($0, 5)], D6=[+($0, 6)]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) +]]> + + + + + "EXISTS"(array(1,2,3), y -> x + y = 4))]]> + + + EXISTS(ARRAY(1, 2, 3), (Y) -> =(+(X, Y), 4)))]) + LogicalValues(tuples=[[{ 0 }]]) ]]> diff --git a/core/src/test/resources/sql/lambda.iq b/core/src/test/resources/sql/lambda.iq index 207543ec77c..82808ff19da 100644 --- a/core/src/test/resources/sql/lambda.iq +++ b/core/src/test/resources/sql/lambda.iq @@ -102,3 +102,46 @@ select "EXISTS"(array[array[1, 2], array[3, 4]], x -> x[1] = 1); (1 row) !ok + +# [CALCITE-6242] Enhance lambda closure parsing +select * + from (select array(1, 2, 3) as arr) as t1 inner join + (select 1 as v) as t2 on "EXISTS"(arr, x -> x = t2.v); ++-----------+---+ +| ARR | V | ++-----------+---+ +| [1, 2, 3] | 1 | ++-----------+---+ +(1 row) + +!ok + +# Nested lambda: x -> (y -> x + y). +# The inner closure 'x' refers to the outer lambda parameter, +# and must not be treated like a table column name. +# For x in (1,2,3) and y in (1,2,3), x + y = 4 holds (e.g. 1+3, 2+2, 3+1). +select "EXISTS"(array(1, 2, 3), x -> "EXISTS"(array(1, 2, 3), y -> x + y = 4)); ++--------+ +| EXPR$0 | ++--------+ +| true | ++--------+ +(1 row) + +!ok + +# Nested lambda with a FROM clause whose column is also named 'x'. +# The inner closure 'x' must resolve to the outer lambda parameter +# (values 1,2,3), NOT to the table column 'x' (value 100). +# If 'x' were resolved as the column, 100 + y = 4 would never hold +# and the result would be false. +select "EXISTS"(array(1, 2, 3), x -> "EXISTS"(array(1, 2, 3), y -> x + y = 4)) as r + from (select 100 as x) as t; ++------+ +| R | ++------+ +| true | ++------+ +(1 row) + +!ok diff --git a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java index 4ab6f776ba4..33385879946 100644 --- a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java +++ b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java @@ -9788,6 +9788,11 @@ private static Consumer> checkWarnings( sql("select 1 || (a, b) ^->^ a + b") .fails(errorMessage2); + + // Nested lambda: inner lambda in a function call within the outer lambda body + sql("select higher_order_func(x -> higher_order_func(y -> x + y, 1), 1) from t") + .ok("SELECT `HIGHER_ORDER_FUNC`(`X` -> `HIGHER_ORDER_FUNC`(`Y` -> (`X` + `Y`), 1), 1)\n" + + "FROM `T`"); } /**