From 8803f1a75cfd4ae1c0ef3981799885ebce027f63 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Mon, 20 Apr 2026 22:33:47 +0800 Subject: [PATCH 01/11] [CALCITE-6451] Improve Nullability Derivation for Intersect and Minus Co-authored-by: Victor Barua --- .../apache/calcite/rel/core/Intersect.java | 11 ++ .../org/apache/calcite/rel/core/Minus.java | 10 + .../org/apache/calcite/rel/core/SetOp.java | 4 + .../rel/rules/IntersectToDistinctRule.java | 27 +++ .../rel/rules/MinusToDistinctRule.java | 4 + .../calcite/sql/fun/SqlStdOperatorTable.java | 12 +- .../apache/calcite/sql/type/ReturnTypes.java | 75 ++++++++ .../apache/calcite/test/RelBuilderTest.java | 179 ++++++++++++++++++ .../apache/calcite/test/RelOptRulesTest.xml | 17 +- core/src/test/resources/sql/planner.iq | 25 ++- .../org/apache/calcite/test/Matchers.java | 13 ++ 11 files changed, 351 insertions(+), 26 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/core/Intersect.java b/core/src/main/java/org/apache/calcite/rel/core/Intersect.java index e6ea2f6c242c..2de728a74279 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Intersect.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Intersect.java @@ -22,7 +22,10 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.metadata.RelMetadataQuery; +import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.util.Util; import java.util.Collections; import java.util.List; @@ -79,4 +82,12 @@ protected Intersect(RelInput input) { dRows *= 0.25; return dRows; } + + @Override protected RelDataType deriveRowType() { + // An output column is only nullable if it is nullable in ALL the inputs. + return ReturnTypes.refineNullabilityForIntersect( + getCluster().getTypeFactory(), + deriveLeastRestrictiveRowType(), + Util.transform(getInputs(), RelNode::getRowType)); + } } diff --git a/core/src/main/java/org/apache/calcite/rel/core/Minus.java b/core/src/main/java/org/apache/calcite/rel/core/Minus.java index 3a68945acc64..c27af940b78f 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Minus.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Minus.java @@ -23,7 +23,9 @@ import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.metadata.RelMdUtil; import org.apache.calcite.rel.metadata.RelMetadataQuery; +import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.type.ReturnTypes; import java.util.Collections; import java.util.List; @@ -60,4 +62,12 @@ protected Minus(RelInput input) { @Override public double estimateRowCount(RelMetadataQuery mq) { return RelMdUtil.getMinusRowCount(mq, this); } + + @Override protected RelDataType deriveRowType() { + // The nullability of the output columns is the same as that of the primary input. + return ReturnTypes.refineNullabilityForExcept( + getCluster().getTypeFactory(), + deriveLeastRestrictiveRowType(), + getInput(0).getRowType()); + } } diff --git a/core/src/main/java/org/apache/calcite/rel/core/SetOp.java b/core/src/main/java/org/apache/calcite/rel/core/SetOp.java index 04e1409e78aa..9af6dd2ca91f 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/SetOp.java +++ b/core/src/main/java/org/apache/calcite/rel/core/SetOp.java @@ -114,6 +114,10 @@ public abstract SetOp copy( } @Override protected RelDataType deriveRowType() { + return deriveLeastRestrictiveRowType(); + } + + protected RelDataType deriveLeastRestrictiveRowType() { final List inputRowTypes = Util.transform(inputs, RelNode::getRowType); final RelDataType rowType = diff --git a/core/src/main/java/org/apache/calcite/rel/rules/IntersectToDistinctRule.java b/core/src/main/java/org/apache/calcite/rel/rules/IntersectToDistinctRule.java index a8d2713e0980..4fb348a9e1b5 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/IntersectToDistinctRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/IntersectToDistinctRule.java @@ -22,8 +22,10 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Intersect; import org.apache.calcite.rel.logical.LogicalIntersect; +import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.tools.RelBuilder.AggCall; import org.apache.calcite.tools.RelBuilderFactory; @@ -176,10 +178,32 @@ public void onMatchAggregatePushdown(RelOptRuleCall call) { final RelOptCluster cluster = intersect.getCluster(); final RexBuilder rexBuilder = cluster.getRexBuilder(); final RelBuilder relBuilder = call.builder(); + List outputFields = intersect.getRowType().getFieldList(); // 1st level aggregate: create an aggregate(col_0, ..., col_n, count(*)), for each branch for (RelNode input : intersect.getInputs()) { relBuilder.push(input); + + // if any of the input fields is non-nullable, the corresponding output field + // is non-nullable this is captured in the type derivation in intersect.getRowType() + // if we know that nulls cannot be present in the output, + // then we can filter them from the inputs before aggregating + ArrayList nullFilters = new ArrayList<>(); + List inputFields = input.getRowType().getFieldList(); + for (int fieldIndex = 0; fieldIndex < outputFields.size(); fieldIndex++) { + RelDataTypeField inputField = inputFields.get(fieldIndex); + if (!outputFields.get(fieldIndex).getType().isNullable() + && inputField.getType().isNullable()) { + nullFilters.add( + rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, + rexBuilder.makeInputRef(input, fieldIndex))); + } + } + + if (!nullFilters.isEmpty()) { + relBuilder.filter(nullFilters); + } + relBuilder.aggregate(relBuilder.groupKey(relBuilder.fields()), relBuilder.countStar(null)); } @@ -206,6 +230,9 @@ public void onMatchAggregatePushdown(RelOptRuleCall call) { // Project all but the last field relBuilder.project(Util.skipLast(relBuilder.fields())); + // ensure the nullabilities of columns in the new relation match those of the input relation + relBuilder.convert(intersect.getRowType(), false); + // the schema for intersect distinct matches that of the relation, // built here with an extra last column for the count, // which is projected out by the final project we added diff --git a/core/src/main/java/org/apache/calcite/rel/rules/MinusToDistinctRule.java b/core/src/main/java/org/apache/calcite/rel/rules/MinusToDistinctRule.java index 7b14f2e94613..59eaf0b60c25 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/MinusToDistinctRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/MinusToDistinctRule.java @@ -159,6 +159,10 @@ public MinusToDistinctRule(Class minusClass, relBuilder.filter(filters.build()); relBuilder.project(Util.first(relBuilder.fields(), originalFieldCnt)); + + // ensure the nullabilities of columns in the new relation match those of the minus output + relBuilder.convert(minus.getRowType(), false); + call.transformTo(relBuilder.build()); } diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java index 2a26a78929cf..156f73132a32 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java @@ -121,16 +121,20 @@ public class SqlStdOperatorTable extends ReflectiveSqlOperatorTable { new SqlSetOperator("UNION ALL", SqlKind.UNION, 12, true); public static final SqlSetOperator EXCEPT = - new SqlSetOperator("EXCEPT", SqlKind.EXCEPT, 12, false); + new SqlSetOperator("EXCEPT", SqlKind.EXCEPT, 12, false, + ReturnTypes.LEAST_RESTRICTIVE_EXCEPT, null, OperandTypes.SET_OP); public static final SqlSetOperator EXCEPT_ALL = - new SqlSetOperator("EXCEPT ALL", SqlKind.EXCEPT, 12, true); + new SqlSetOperator("EXCEPT ALL", SqlKind.EXCEPT, 12, true, + ReturnTypes.LEAST_RESTRICTIVE_EXCEPT, null, OperandTypes.SET_OP); public static final SqlSetOperator INTERSECT = - new SqlSetOperator("INTERSECT", SqlKind.INTERSECT, 14, false); + new SqlSetOperator("INTERSECT", SqlKind.INTERSECT, 14, false, + ReturnTypes.LEAST_RESTRICTIVE_INTERSECT, null, OperandTypes.SET_OP); public static final SqlSetOperator INTERSECT_ALL = - new SqlSetOperator("INTERSECT ALL", SqlKind.INTERSECT, 14, true); + new SqlSetOperator("INTERSECT ALL", SqlKind.INTERSECT, 14, true, + ReturnTypes.LEAST_RESTRICTIVE_INTERSECT, null, OperandTypes.SET_OP); /** * The {@code MULTISET UNION DISTINCT} operator. diff --git a/core/src/main/java/org/apache/calcite/sql/type/ReturnTypes.java b/core/src/main/java/org/apache/calcite/sql/type/ReturnTypes.java index 27f1a6fc72a7..7c8680e64336 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/ReturnTypes.java +++ b/core/src/main/java/org/apache/calcite/sql/type/ReturnTypes.java @@ -608,6 +608,81 @@ public static SqlCall stripSeparator(SqlCall call) { .leastRestrictive(opBinding.collectOperandTypes()); } + /** + * Refines the nullability of {@code base} using INTERSECT semantics: a + * column is NOT NULL if it is NOT NULL in at least one of the + * {@code inputTypes} (AND-semantics across inputs). + */ + public static RelDataType refineNullabilityForIntersect( + RelDataTypeFactory typeFactory, + RelDataType base, + List inputTypes) { + final RelDataTypeFactory.Builder builder = + new RelDataTypeFactory.Builder(typeFactory); + final List outputFields = base.getFieldList(); + for (int i = 0; i < outputFields.size(); i++) { + boolean nullable = true; + for (RelDataType inputType : inputTypes) { + nullable &= inputType.getFieldList().get(i).getType().isNullable(); + } + builder.add(outputFields.get(i)).nullable(nullable); + } + return builder.build(); + } + + /** + * Type-inference strategy for INTERSECT. Computes the least restrictive row + * type across all inputs, then refines nullability: a column is NOT NULL if + * it is NOT NULL in at least one input (AND semantics across inputs). + */ + public static final SqlReturnTypeInference LEAST_RESTRICTIVE_INTERSECT = + andThen(SqlTypeTransforms.FROM_MEASURE_IF::apply, opBinding -> { + final List inputTypes = opBinding.collectOperandTypes(); + final RelDataType base = + opBinding.getTypeFactory().leastRestrictive(inputTypes); + if (base == null) { + return null; + } + return refineNullabilityForIntersect(opBinding.getTypeFactory(), base, inputTypes); + }); + + /** + * Refines the nullability of {@code base} using EXCEPT/MINUS semantics: a + * column's nullability matches that of the first (primary) input. + */ + public static RelDataType refineNullabilityForExcept( + RelDataTypeFactory typeFactory, + RelDataType base, + RelDataType primaryInputType) { + final RelDataTypeFactory.Builder builder = + new RelDataTypeFactory.Builder(typeFactory); + final List outputFields = base.getFieldList(); + final List primaryInputFields = + primaryInputType.getFieldList(); + for (int i = 0; i < outputFields.size(); i++) { + builder.add(outputFields.get(i)) + .nullable(primaryInputFields.get(i).getType().isNullable()); + } + return builder.build(); + } + + /** + * Type-inference strategy for EXCEPT/MINUS. Computes the least restrictive + * row type across all inputs, then refines nullability: a column's + * nullability matches that of the first (primary) input. + */ + public static final SqlReturnTypeInference LEAST_RESTRICTIVE_EXCEPT = + andThen(SqlTypeTransforms.FROM_MEASURE_IF::apply, opBinding -> { + final List inputTypes = opBinding.collectOperandTypes(); + final RelDataType base = + opBinding.getTypeFactory().leastRestrictive(inputTypes); + if (base == null) { + return null; + } + return refineNullabilityForExcept( + opBinding.getTypeFactory(), base, inputTypes.get(0)); + }); + /** * Type-inference strategy for NVL2 function. It returns the least restrictive type * between the second and third operands. diff --git a/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java b/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java index b54e08e26d4f..a09d8f2c3d84 100644 --- a/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java @@ -131,6 +131,7 @@ import static org.apache.calcite.test.Matchers.hasExpandedTree; import static org.apache.calcite.test.Matchers.hasFieldNames; import static org.apache.calcite.test.Matchers.hasHints; +import static org.apache.calcite.test.Matchers.hasRelDataType; import static org.apache.calcite.test.Matchers.hasTree; import static org.hamcrest.CoreMatchers.allOf; @@ -2346,6 +2347,72 @@ private static RelNode groupIdRel(RelBuilder builder, boolean extra) { assertThat(root, hasTree(expected)); } + /** Test case for + * [CALCITE-6451] + * Improve Nullability Derivation for Intersect and Minus. */ + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testUnionTypeDerivation(boolean all) { + final RelBuilder builder = RelBuilder.create(config().build()); + + RelDataType input1RowType = + new RelDataTypeFactory.Builder(builder.getTypeFactory()) + .add("a", SqlTypeName.BIGINT) + .nullable(false) + .add("b", SqlTypeName.BIGINT) + .nullable(false) + .add("c", SqlTypeName.BIGINT) + .nullable(true) + .add("d", SqlTypeName.BIGINT) + .nullable(true) + .build(); + + RelDataType input2RowType = + new RelDataTypeFactory.Builder(builder.getTypeFactory()) + .add("a", SqlTypeName.BIGINT) + .nullable(false) + .add("b", SqlTypeName.BIGINT) + .nullable(false) + .add("c", SqlTypeName.BIGINT) + .nullable(false) + .add("d", SqlTypeName.BIGINT) + .nullable(false) + .build(); + + RelDataType input3RowType = + new RelDataTypeFactory.Builder(builder.getTypeFactory()) + .add("a", SqlTypeName.BIGINT) + .nullable(false) + .add("b", SqlTypeName.BIGINT) + .nullable(true) + .add("c", SqlTypeName.BIGINT) + .nullable(false) + .add("d", SqlTypeName.BIGINT) + .nullable(true) + .build(); + + RelNode root = + builder + .values(input1RowType) + .values(input2RowType) + .values(input3RowType) + .union(all, 3) + .build(); + + RelDataType expectedRowType = + new RelDataTypeFactory.Builder(builder.getTypeFactory()) + .add("a", SqlTypeName.BIGINT) + .nullable(false) + .add("b", SqlTypeName.BIGINT) + .nullable(true) + .add("c", SqlTypeName.BIGINT) + .nullable(true) + .add("d", SqlTypeName.BIGINT) + .nullable(true) + .build(); + assertThat(root.getRowType(), hasRelDataType(expectedRowType)); + } + /** Test case for * [CALCITE-1522] * Fix error message for SetOp with incompatible args. */ @@ -2550,6 +2617,69 @@ private static RelNode groupIdRel(RelBuilder builder, boolean extra) { assertThat(root, hasTree(expected)); } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testIntersectTypeDerivation(boolean all) { + final RelBuilder builder = RelBuilder.create(config().build()); + + RelDataType input1RowType = + new RelDataTypeFactory.Builder(builder.getTypeFactory()) + .add("a", SqlTypeName.BIGINT) + .nullable(false) + .add("b", SqlTypeName.BIGINT) + .nullable(false) + .add("c", SqlTypeName.BIGINT) + .nullable(true) + .add("d", SqlTypeName.BIGINT) + .nullable(true) + .build(); + + RelDataType input2RowType = + new RelDataTypeFactory.Builder(builder.getTypeFactory()) + .add("a", SqlTypeName.BIGINT) + .nullable(true) + .add("b", SqlTypeName.BIGINT) + .nullable(true) + .add("c", SqlTypeName.BIGINT) + .nullable(true) + .add("d", SqlTypeName.BIGINT) + .nullable(true) + .build(); + + RelDataType input3RowType = + new RelDataTypeFactory.Builder(builder.getTypeFactory()) + .add("a", SqlTypeName.BIGINT) + .nullable(false) + .add("b", SqlTypeName.BIGINT) + .nullable(true) + .add("c", SqlTypeName.BIGINT) + .nullable(false) + .add("d", SqlTypeName.BIGINT) + .nullable(true) + .build(); + + RelNode root = + builder + .values(input1RowType) + .values(input2RowType) + .values(input3RowType) + .intersect(all, 3) + .build(); + + RelDataType expectedRowType = + new RelDataTypeFactory.Builder(builder.getTypeFactory()) + .add("a", SqlTypeName.BIGINT) + .nullable(false) + .add("b", SqlTypeName.BIGINT) + .nullable(false) + .add("c", SqlTypeName.BIGINT) + .nullable(false) + .add("d", SqlTypeName.BIGINT) + .nullable(true) + .build(); + assertThat(root.getRowType(), hasRelDataType(expectedRowType)); + } + @Test void testExcept() { // Equivalent SQL: // SELECT empno FROM emp @@ -2577,6 +2707,55 @@ private static RelNode groupIdRel(RelBuilder builder, boolean extra) { assertThat(root, hasTree(expected)); } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testExceptTypeDerivation(boolean all) { + final RelBuilder builder = RelBuilder.create(config().build()); + + RelDataType primaryRowType = + new RelDataTypeFactory.Builder(builder.getTypeFactory()) + .add("a", SqlTypeName.BIGINT) + .nullable(false) + .add("b", SqlTypeName.BIGINT) + .nullable(false) + .add("c", SqlTypeName.BIGINT) + .nullable(true) + .add("d", SqlTypeName.BIGINT) + .nullable(true) + .build(); + + RelDataType secondaryRowType = + new RelDataTypeFactory.Builder(builder.getTypeFactory()) + .add("a", SqlTypeName.BIGINT) + .nullable(false) + .add("b", SqlTypeName.BIGINT) + .nullable(true) + .add("c", SqlTypeName.BIGINT) + .nullable(false) + .add("d", SqlTypeName.BIGINT) + .nullable(true) + .build(); + + RelNode root = + builder.values(primaryRowType) + .values(secondaryRowType) + .minus(all) + .build(); + + RelDataType expectedRowType = + new RelDataTypeFactory.Builder(builder.getTypeFactory()) + .add("a", SqlTypeName.BIGINT) + .nullable(false) + .add("b", SqlTypeName.BIGINT) + .nullable(false) + .add("c", SqlTypeName.BIGINT) + .nullable(true) + .add("d", SqlTypeName.BIGINT) + .nullable(true) + .build(); + assertThat(root.getRowType(), hasRelDataType(expectedRowType)); + } + /** Tests building a simple join. Also checks {@link RelBuilder#size()} * at every step. */ @Test void testJoin() { diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index ece2ad5258aa..0043fedfd081 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -7302,15 +7302,14 @@ LogicalIntersect(all=[false]) LogicalAggregate(group=[{0}]) LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[semi]) LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[semi]) - LogicalProject(ENAME=[CAST($0):VARCHAR]) + LogicalProject(ENAME=[CAST($0):VARCHAR NOT NULL]) LogicalProject(ENAME=[$1]) LogicalFilter(condition=[=($7, 10)]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) - LogicalProject(ENAME=[CAST($0):VARCHAR]) - LogicalProject(DEPTNO=[CAST($7):VARCHAR NOT NULL]) - LogicalFilter(condition=[OR(=($1, 'a'), =($1, 'b'))]) - LogicalTableScan(table=[[CATALOG, SALES, EMP]]) - LogicalProject(ENAME=[CAST($0):VARCHAR]) + LogicalProject(DEPTNO=[CAST($7):VARCHAR NOT NULL]) + LogicalFilter(condition=[OR(=($1, 'a'), =($1, 'b'))]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) + LogicalProject(ENAME=[CAST($0):VARCHAR NOT NULL]) LogicalProject(ENAME=[$1]) LogicalTableScan(table=[[CATALOG, SALES, EMPNULLABLES]]) ]]> @@ -10436,10 +10435,10 @@ LogicalMinus(all=[false]) hasFieldNames(String fieldNames) { } }; } + + /** + * Creates a Matcher that matches a {@link RelDataType} if its + * {@link RelDataType#getFullTypeString()} is equal to that of the given {@code relDataType}. + */ + public static Matcher hasRelDataType(RelDataType relDataType) { + return compose( + IsEqual.equalTo(relDataType.getFullTypeString()), + RelDataType::getFullTypeString); + } + /** * Creates a Matcher that matches a {@link RelNode} if its string * representation, after converting Windows-style line endings ("\r\n") From 3aa4602738886b8e3b79e75c1cc61f17185ede01 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Tue, 21 Apr 2026 09:22:13 +0800 Subject: [PATCH 02/11] Fix CI Co-authored-by: Victor Barua --- .../src/main/java/org/apache/calcite/sql/SqlSetOperator.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/sql/SqlSetOperator.java b/core/src/main/java/org/apache/calcite/sql/SqlSetOperator.java index 6d0f48c4f417..a776a180432f 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlSetOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlSetOperator.java @@ -21,9 +21,12 @@ import org.apache.calcite.sql.type.SqlOperandTypeChecker; import org.apache.calcite.sql.type.SqlOperandTypeInference; import org.apache.calcite.sql.type.SqlReturnTypeInference; + import org.apache.calcite.sql.validate.SqlValidator; import org.apache.calcite.sql.validate.SqlValidatorScope; +import org.checkerframework.checker.nullness.qual.Nullable; + /** * SqlSetOperator represents a relational set theory operator (UNION, INTERSECT, * MINUS). These are binary operators, but with an extra boolean attribute @@ -59,7 +62,7 @@ public SqlSetOperator( int prec, boolean all, SqlReturnTypeInference returnTypeInference, - SqlOperandTypeInference operandTypeInference, + @Nullable SqlOperandTypeInference operandTypeInference, SqlOperandTypeChecker operandTypeChecker) { super( name, From 623720aaae7a612bf4d5d97149b1e6aa777115c4 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Tue, 21 Apr 2026 09:32:22 +0800 Subject: [PATCH 03/11] Fix CI Co-authored-by: Victor Barua --- core/src/main/java/org/apache/calcite/sql/SqlSetOperator.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/sql/SqlSetOperator.java b/core/src/main/java/org/apache/calcite/sql/SqlSetOperator.java index a776a180432f..0ce69276f71d 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlSetOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlSetOperator.java @@ -21,7 +21,6 @@ import org.apache.calcite.sql.type.SqlOperandTypeChecker; import org.apache.calcite.sql.type.SqlOperandTypeInference; import org.apache.calcite.sql.type.SqlReturnTypeInference; - import org.apache.calcite.sql.validate.SqlValidator; import org.apache.calcite.sql.validate.SqlValidatorScope; From 0566daf5d6d50b11ad0705b688898dc138826f0f Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Thu, 30 Apr 2026 23:39:57 +0800 Subject: [PATCH 04/11] Addressed --- .../apache/calcite/rel/rules/IntersectToDistinctRule.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/IntersectToDistinctRule.java b/core/src/main/java/org/apache/calcite/rel/rules/IntersectToDistinctRule.java index 4fb348a9e1b5..abcd57c6731d 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/IntersectToDistinctRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/IntersectToDistinctRule.java @@ -140,6 +140,9 @@ public void onMatchAggregateOnUnion(RelOptRuleCall call) { } relBuilder.filter(filters); + // ensure the nullabilities of columns in the new relation match those of the input relation + relBuilder.convert(intersect.getRowType(), false); + // Project all but the last added field (e.g. count_i{n}) relBuilder.project(skipLast(relBuilder.fields(), branchCount)); call.transformTo(relBuilder.build()); @@ -200,10 +203,6 @@ public void onMatchAggregatePushdown(RelOptRuleCall call) { } } - if (!nullFilters.isEmpty()) { - relBuilder.filter(nullFilters); - } - relBuilder.aggregate(relBuilder.groupKey(relBuilder.fields()), relBuilder.countStar(null)); } From f55065bd91e50da8a06a64381ce26a0bbe258acd Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Fri, 1 May 2026 10:41:58 +0800 Subject: [PATCH 05/11] Addressed --- .../apache/calcite/rel/rules/IntersectToDistinctRule.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/IntersectToDistinctRule.java b/core/src/main/java/org/apache/calcite/rel/rules/IntersectToDistinctRule.java index abcd57c6731d..bdbbb6ee36b8 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/IntersectToDistinctRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/IntersectToDistinctRule.java @@ -140,11 +140,12 @@ public void onMatchAggregateOnUnion(RelOptRuleCall call) { } relBuilder.filter(filters); + // Project all but the last added field (e.g. count_i{n}) + relBuilder.project(skipLast(relBuilder.fields(), branchCount)); + // ensure the nullabilities of columns in the new relation match those of the input relation relBuilder.convert(intersect.getRowType(), false); - // Project all but the last added field (e.g. count_i{n}) - relBuilder.project(skipLast(relBuilder.fields(), branchCount)); call.transformTo(relBuilder.build()); } From 051caa25a34ba3d9acb13d254b2d5a4d340c8a14 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Fri, 1 May 2026 10:50:31 +0800 Subject: [PATCH 06/11] Addressed --- .../apache/calcite/rel/rules/IntersectToDistinctRule.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/IntersectToDistinctRule.java b/core/src/main/java/org/apache/calcite/rel/rules/IntersectToDistinctRule.java index bdbbb6ee36b8..24a80173b47c 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/IntersectToDistinctRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/IntersectToDistinctRule.java @@ -192,7 +192,7 @@ public void onMatchAggregatePushdown(RelOptRuleCall call) { // is non-nullable this is captured in the type derivation in intersect.getRowType() // if we know that nulls cannot be present in the output, // then we can filter them from the inputs before aggregating - ArrayList nullFilters = new ArrayList<>(); + List nullFilters = new ArrayList<>(); List inputFields = input.getRowType().getFieldList(); for (int fieldIndex = 0; fieldIndex < outputFields.size(); fieldIndex++) { RelDataTypeField inputField = inputFields.get(fieldIndex); @@ -203,6 +203,9 @@ public void onMatchAggregatePushdown(RelOptRuleCall call) { rexBuilder.makeInputRef(input, fieldIndex))); } } + if (!nullFilters.isEmpty()) { + relBuilder.filter(nullFilters); + } relBuilder.aggregate(relBuilder.groupKey(relBuilder.fields()), relBuilder.countStar(null)); From 46141d164a552b7be66418ad5cdae0791c959f4a Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Fri, 1 May 2026 10:53:56 +0800 Subject: [PATCH 07/11] Addressed --- .../rel/rules/IntersectToDistinctRule.java | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/IntersectToDistinctRule.java b/core/src/main/java/org/apache/calcite/rel/rules/IntersectToDistinctRule.java index 24a80173b47c..c254f9d3e71e 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/IntersectToDistinctRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/IntersectToDistinctRule.java @@ -188,29 +188,6 @@ public void onMatchAggregatePushdown(RelOptRuleCall call) { for (RelNode input : intersect.getInputs()) { relBuilder.push(input); - // if any of the input fields is non-nullable, the corresponding output field - // is non-nullable this is captured in the type derivation in intersect.getRowType() - // if we know that nulls cannot be present in the output, - // then we can filter them from the inputs before aggregating - List nullFilters = new ArrayList<>(); - List inputFields = input.getRowType().getFieldList(); - for (int fieldIndex = 0; fieldIndex < outputFields.size(); fieldIndex++) { - RelDataTypeField inputField = inputFields.get(fieldIndex); - if (!outputFields.get(fieldIndex).getType().isNullable() - && inputField.getType().isNullable()) { - nullFilters.add( - rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, - rexBuilder.makeInputRef(input, fieldIndex))); - } - } - if (!nullFilters.isEmpty()) { - relBuilder.filter(nullFilters); - } - - relBuilder.aggregate(relBuilder.groupKey(relBuilder.fields()), - relBuilder.countStar(null)); - } - // create a union above all the branches final int branchCount = intersect.getInputs().size(); relBuilder.union(true, branchCount); From 611c7b5fff226ca6f04cf751ee53c2569873b86c Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Fri, 1 May 2026 10:58:31 +0800 Subject: [PATCH 08/11] Addressed --- .../org/apache/calcite/rel/rules/IntersectToDistinctRule.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/IntersectToDistinctRule.java b/core/src/main/java/org/apache/calcite/rel/rules/IntersectToDistinctRule.java index c254f9d3e71e..b169bfad170f 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/IntersectToDistinctRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/IntersectToDistinctRule.java @@ -187,6 +187,9 @@ public void onMatchAggregatePushdown(RelOptRuleCall call) { // 1st level aggregate: create an aggregate(col_0, ..., col_n, count(*)), for each branch for (RelNode input : intersect.getInputs()) { relBuilder.push(input); + relBuilder.aggregate(relBuilder.groupKey(relBuilder.fields()), + relBuilder.countStar(null)); + } // create a union above all the branches final int branchCount = intersect.getInputs().size(); From 4f69274a366d19e96acf896429ae94e3d0bd34ca Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Fri, 1 May 2026 11:08:12 +0800 Subject: [PATCH 09/11] Addressed --- .../org/apache/calcite/rel/rules/IntersectToDistinctRule.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/IntersectToDistinctRule.java b/core/src/main/java/org/apache/calcite/rel/rules/IntersectToDistinctRule.java index b169bfad170f..246fcc271dbc 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/IntersectToDistinctRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/IntersectToDistinctRule.java @@ -25,7 +25,6 @@ import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexNode; -import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.tools.RelBuilder.AggCall; import org.apache.calcite.tools.RelBuilderFactory; From 66b93f217fce1a45dc52b7ba69b56b04ac06d678 Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Fri, 1 May 2026 11:32:29 +0800 Subject: [PATCH 10/11] Addressed --- .../org/apache/calcite/rel/rules/IntersectToDistinctRule.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/IntersectToDistinctRule.java b/core/src/main/java/org/apache/calcite/rel/rules/IntersectToDistinctRule.java index 246fcc271dbc..1531847bff80 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/IntersectToDistinctRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/IntersectToDistinctRule.java @@ -181,7 +181,6 @@ public void onMatchAggregatePushdown(RelOptRuleCall call) { final RelOptCluster cluster = intersect.getCluster(); final RexBuilder rexBuilder = cluster.getRexBuilder(); final RelBuilder relBuilder = call.builder(); - List outputFields = intersect.getRowType().getFieldList(); // 1st level aggregate: create an aggregate(col_0, ..., col_n, count(*)), for each branch for (RelNode input : intersect.getInputs()) { From e096f40c5c32555d11ff8f6ec770e77817c7057d Mon Sep 17 00:00:00 2001 From: xiedeyantu Date: Fri, 1 May 2026 11:46:20 +0800 Subject: [PATCH 11/11] Addressed --- .../org/apache/calcite/rel/rules/IntersectToDistinctRule.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/rel/rules/IntersectToDistinctRule.java b/core/src/main/java/org/apache/calcite/rel/rules/IntersectToDistinctRule.java index 1531847bff80..394f8e088551 100644 --- a/core/src/main/java/org/apache/calcite/rel/rules/IntersectToDistinctRule.java +++ b/core/src/main/java/org/apache/calcite/rel/rules/IntersectToDistinctRule.java @@ -22,7 +22,6 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Intersect; import org.apache.calcite.rel.logical.LogicalIntersect; -import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexNode; import org.apache.calcite.tools.RelBuilder;