diff --git a/babel/src/main/codegen/config.fmpp b/babel/src/main/codegen/config.fmpp index 001bdf2e1034..f7d3e7fa8089 100644 --- a/babel/src/main/codegen/config.fmpp +++ b/babel/src/main/codegen/config.fmpp @@ -619,6 +619,7 @@ data: { includeIntervalWithoutQualifier: true includeStarExclude: true includeSelectBy: true + includeDistinctOn: true } } diff --git a/babel/src/test/java/org/apache/calcite/test/BabelTest.java b/babel/src/test/java/org/apache/calcite/test/BabelTest.java index 6970280ed436..f47ffd3b621e 100644 --- a/babel/src/test/java/org/apache/calcite/test/BabelTest.java +++ b/babel/src/test/java/org/apache/calcite/test/BabelTest.java @@ -521,6 +521,35 @@ private void checkSqlResult(String funLibrary, String query, String result) { .returns(result); } + /** Test case for + * [CALCITE-5406] + * Support the SELECT DISTINCT ON statement. */ + @Test void testDistinctOn() { + final SqlValidatorFixture v = Fixtures.forValidator() + .withParserConfig(c -> c.withParserFactory(SqlBabelParserImpl.FACTORY)) + .withConformance(SqlConformanceEnum.BABEL); + + // Basic DISTINCT ON + v.withSql("select distinct on (deptno) empno, ename from emp order by deptno, empno") + .ok(); + + // DISTINCT ON with multiple columns + v.withSql("select distinct on (deptno, job) empno, ename from emp order by deptno, job, empno") + .ok(); + + // DISTINCT ON with expression + v.withSql("select distinct on (deptno) empno, sal * 12 as annual_sal " + + "from emp order by deptno, sal desc").ok(); + + // DISTINCT ON without ORDER BY should fail + v.withSql("^select distinct on (deptno) empno from emp^") + .fails("SELECT DISTINCT ON requires an ORDER BY clause"); + + // DISTINCT ON with ORDER BY mismatch should fail + v.withSql("select distinct on (deptno) empno from emp order by ^empno^") + .fails("SELECT DISTINCT ON expressions must match initial ORDER BY expressions"); + } + /** Test case for * [CALCITE-7337] * Add age function (enabled in PostgreSQL library). */ diff --git a/babel/src/test/resources/sql/select.iq b/babel/src/test/resources/sql/select.iq index c969ad76559e..75d03a39af1f 100755 --- a/babel/src/test/resources/sql/select.iq +++ b/babel/src/test/resources/sql/select.iq @@ -286,4 +286,82 @@ select d1.* except(d1.dname) from dept d1 except(select d2.* except(d2.dname) fr !ok +# [CALCITE-5406] Support the SELECT DISTINCT ON statement for PostgreSQL dialect + +# Test basic DISTINCT ON +SELECT DISTINCT ON (deptno) empno, ename, deptno +FROM emp +ORDER BY deptno, empno; ++-------+-------+--------+ +| EMPNO | ENAME | DEPTNO | ++-------+-------+--------+ +| 7782 | CLARK | 10 | +| 7369 | SMITH | 20 | +| 7499 | ALLEN | 30 | ++-------+-------+--------+ +(3 rows) + +!ok + +# Test DISTINCT ON with descending order +SELECT DISTINCT ON (deptno) empno, ename, sal, deptno +FROM emp +ORDER BY deptno, sal DESC, empno; ++-------+-------+---------+--------+ +| EMPNO | ENAME | SAL | DEPTNO | ++-------+-------+---------+--------+ +| 7839 | KING | 5000.00 | 10 | +| 7788 | SCOTT | 3000.00 | 20 | +| 7698 | BLAKE | 2850.00 | 30 | ++-------+-------+---------+--------+ +(3 rows) + +!ok + +# Test DISTINCT ON with multiple columns +SELECT DISTINCT ON (deptno, job) empno, ename, deptno, job +FROM emp +ORDER BY deptno, job, empno; ++-------+--------+--------+-----------+ +| EMPNO | ENAME | DEPTNO | JOB | ++-------+--------+--------+-----------+ +| 7934 | MILLER | 10 | CLERK | +| 7782 | CLARK | 10 | MANAGER | +| 7839 | KING | 10 | PRESIDENT | +| 7788 | SCOTT | 20 | ANALYST | +| 7369 | SMITH | 20 | CLERK | +| 7566 | JONES | 20 | MANAGER | +| 7900 | JAMES | 30 | CLERK | +| 7698 | BLAKE | 30 | MANAGER | +| 7499 | ALLEN | 30 | SALESMAN | ++-------+--------+--------+-----------+ +(9 rows) + +!ok + +# Test DISTINCT ON with expression +SELECT DISTINCT ON (deptno) empno, ename, sal * 12 AS annual_sal, deptno +FROM emp +ORDER BY deptno, sal DESC, empno; ++-------+-------+------------+--------+ +| EMPNO | ENAME | ANNUAL_SAL | DEPTNO | ++-------+-------+------------+--------+ +| 7788 | SCOTT | 36000.00 | 20 | +| 7839 | KING | 60000.00 | 10 | +| 7698 | BLAKE | 34200.00 | 30 | ++-------+-------+------------+--------+ +(3 rows) + +!ok + +# Test DISTINCT ON unparse +SELECT DISTINCT ON (deptno) empno, ename, deptno +FROM emp +ORDER BY deptno, empno; + +SELECT DISTINCT ON ("DEPTNO") "EMP"."EMPNO", "EMP"."ENAME", "EMP"."DEPTNO" +FROM "scott"."EMP" AS "EMP" +ORDER BY "DEPTNO", "EMPNO" +!explain-validated-on all + # End select.iq diff --git a/core/src/main/codegen/config.fmpp b/core/src/main/codegen/config.fmpp index 73d981bf3934..d8286c4ce1d0 100644 --- a/core/src/main/codegen/config.fmpp +++ b/core/src/main/codegen/config.fmpp @@ -40,8 +40,9 @@ data: { # FMPP will use the declaration from default_config.fmpp. parser: { # Generated parser implementation package and class name. - package: "org.apache.calcite.sql.parser.impl", - class: "SqlParserImpl", + package: "org.apache.calcite.sql.parser.impl" + class: "SqlParserImpl" + includeDistinctOn: true # List of files in @includes directory that have parser method # implementations for parsing custom SQL statements, literals or types diff --git a/core/src/main/codegen/default_config.fmpp b/core/src/main/codegen/default_config.fmpp index 56d17b82798b..9cb054233719 100644 --- a/core/src/main/codegen/default_config.fmpp +++ b/core/src/main/codegen/default_config.fmpp @@ -461,4 +461,5 @@ parser: { includeParsingStringLiteralAsArrayLiteral: false includeIntervalWithoutQualifier: false includeStarExclude: false + includeDistinctOn: false } diff --git a/core/src/main/codegen/templates/Parser.jj b/core/src/main/codegen/templates/Parser.jj index 16b80218e59d..feaef05dcc17 100644 --- a/core/src/main/codegen/templates/Parser.jj +++ b/core/src/main/codegen/templates/Parser.jj @@ -1021,6 +1021,24 @@ List FunctionParameterList(ExprContext exprContext) : } } +void AllOrDistinctOrDistinctOn(List keywords, List distinctOnList) : +{ + final Span s; + final SqlNodeList distinctOn; +} +{ + { keywords.add(SqlSelectKeyword.ALL.symbol(getPos())); } +| + { s = span(); } + ( + + distinctOn = ExpressionCommaList(s, ExprContext.ACCEPT_SUB_QUERY) + + { distinctOnList.addAll(distinctOn.getList()); } + )? + { keywords.add(SqlSelectKeyword.DISTINCT.symbol(s.end(this))); } +} + SqlLiteral AllOrDistinct() : { } @@ -1371,6 +1389,7 @@ SqlSelect SqlSelect() : final SqlNode qualify; final SqlNodeList by; final List hints = new ArrayList(); + final List distinctOnList = new ArrayList(); final Span s; } { @@ -1382,9 +1401,15 @@ SqlSelect SqlSelect() : keywords.add(SqlSelectKeyword.STREAM.symbol(getPos())); } )? +<#if parser.includeDistinctOn!default.parser.includeDistinctOn> + ( + AllOrDistinctOrDistinctOn(keywords, distinctOnList) + )? +<#else> ( keyword = AllOrDistinct() { keywords.add(keyword); } )? + { keywordList = new SqlNodeList(keywords, s.addAll(keywords).pos()); } @@ -1413,10 +1438,14 @@ SqlSelect SqlSelect() : } ) { + final SqlNodeList distinctOn = distinctOnList.isEmpty() + ? null + : new SqlNodeList(distinctOnList, Span.of(distinctOnList).pos()); final SqlSelect select = new SqlSelect(s.end(this), keywordList, new SqlNodeList(selectList, Span.of(selectList).pos()), fromClause, where, groupBy, having, windowDecls, qualify, - null, null, null, new SqlNodeList(hints, getPos())); + null, null, null, new SqlNodeList(hints, getPos()), + distinctOn); SqlByRewriter.rewrite(select, by); return select; } diff --git a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java index baf5fca10e04..a7e82b15f11b 100644 --- a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java +++ b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java @@ -505,6 +505,15 @@ ExInst intervalFieldExceedsPrecision(Number a0, @BaseMessage("QUALIFY expression ''{0}'' must contain a window function") ExInst qualifyExpressionMustContainWindowFunction(String a0); + @BaseMessage("SELECT DISTINCT ON is not supported under the current SQL conformance level") + ExInst distinctOnNotAllowed(); + + @BaseMessage("SELECT DISTINCT ON requires an ORDER BY clause") + ExInst distinctOnRequiresOrderBy(); + + @BaseMessage("SELECT DISTINCT ON expressions must match initial ORDER BY expressions") + ExInst distinctOnOrderByMismatch(); + @BaseMessage("ROW/RANGE not allowed with RANK, DENSE_RANK, ROW_NUMBER, PERCENTILE_CONT/DISC or LAG/LEAD functions") ExInst rankWithFrame(); diff --git a/core/src/main/java/org/apache/calcite/sql/SqlSelect.java b/core/src/main/java/org/apache/calcite/sql/SqlSelect.java index 0faa1d024f57..14a8d63711c8 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlSelect.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlSelect.java @@ -43,6 +43,7 @@ public class SqlSelect extends SqlCall { public static final int WHERE_OPERAND = 3; public static final int HAVING_OPERAND = 5; public static final int QUALIFY_OPERAND = 7; + public static final int DISTINCT_ON_OPERAND = 12; SqlNodeList keywordList; SqlNodeList selectList; @@ -56,6 +57,7 @@ public class SqlSelect extends SqlCall { @Nullable SqlNode offset; @Nullable SqlNode fetch; @Nullable SqlNodeList hints; + @Nullable SqlNodeList distinctOn; boolean hasByClause; //~ Constructors ----------------------------------------------------------- @@ -72,7 +74,8 @@ public SqlSelect(SqlParserPos pos, @Nullable SqlNodeList orderBy, @Nullable SqlNode offset, @Nullable SqlNode fetch, - @Nullable SqlNodeList hints) { + @Nullable SqlNodeList hints, + @Nullable SqlNodeList distinctOn) { super(pos); this.keywordList = requireNonNull(keywordList != null ? keywordList : new SqlNodeList(pos)); @@ -88,10 +91,29 @@ public SqlSelect(SqlParserPos pos, this.offset = offset; this.fetch = fetch; this.hints = hints; + this.distinctOn = distinctOn; this.hasByClause = false; } - /** deprecated, without {@code qualify}. */ + /** Constructor without {@code distinctOn}; distinctOn defaults to null. */ + public SqlSelect(SqlParserPos pos, + @Nullable SqlNodeList keywordList, + SqlNodeList selectList, + @Nullable SqlNode from, + @Nullable SqlNode where, + @Nullable SqlNodeList groupBy, + @Nullable SqlNode having, + @Nullable SqlNodeList windowDecls, + @Nullable SqlNode qualify, + @Nullable SqlNodeList orderBy, + @Nullable SqlNode offset, + @Nullable SqlNode fetch, + @Nullable SqlNodeList hints) { + this(pos, keywordList, selectList, from, where, groupBy, having, + windowDecls, qualify, orderBy, offset, fetch, hints, null); + } + + /** deprecated, without {@code qualify} and {@code distinctOn}. */ @Deprecated // to be removed before 2.0 public SqlSelect(SqlParserPos pos, @Nullable SqlNodeList keywordList, @@ -106,7 +128,7 @@ public SqlSelect(SqlParserPos pos, @Nullable SqlNode fetch, @Nullable SqlNodeList hints) { this(pos, keywordList, selectList, from, where, groupBy, having, - windowDecls, null, orderBy, offset, fetch, hints); + windowDecls, null, orderBy, offset, fetch, hints, null); } //~ Methods ---------------------------------------------------------------- @@ -122,7 +144,8 @@ public SqlSelect(SqlParserPos pos, @SuppressWarnings("nullness") @Override public List getOperandList() { return ImmutableNullableList.of(keywordList, selectList, from, where, - groupBy, having, windowDecls, qualify, orderBy, offset, fetch, hints); + groupBy, having, windowDecls, qualify, orderBy, offset, fetch, hints, + distinctOn); } @Override public void setOperand(int i, @Nullable SqlNode operand) { @@ -160,6 +183,12 @@ public SqlSelect(SqlParserPos pos, case 10: fetch = operand; break; + case 11: + hints = (SqlNodeList) operand; + break; + case 12: + distinctOn = (SqlNodeList) operand; + break; default: throw new AssertionError(i); } @@ -327,4 +356,16 @@ public boolean hasWhere() { public boolean isKeywordPresent(SqlSelectKeyword targetKeyWord) { return getModifierNode(targetKeyWord) != null; } + + public boolean isDistinctOn() { + return distinctOn != null && !distinctOn.isEmpty(); + } + + public @Nullable SqlNodeList getDistinctOn() { + return distinctOn; + } + + public void setDistinctOn(@Nullable SqlNodeList distinctOn) { + this.distinctOn = distinctOn; + } } diff --git a/core/src/main/java/org/apache/calcite/sql/SqlSelectOperator.java b/core/src/main/java/org/apache/calcite/sql/SqlSelectOperator.java index b8db2d8f5ab8..02ee42137865 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlSelectOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlSelectOperator.java @@ -37,14 +37,19 @@ *

Operands are: * *

    - *
  • 0: distinct ({@link SqlLiteral})
    • + *
  • 0: keywordList ({@link SqlNodeList})
    • *
  • 1: selectClause ({@link SqlNodeList})
    • *
  • 2: fromClause ({@link SqlCall} to "join" operator)
    • *
  • 3: whereClause ({@link SqlNode})
    • - *
  • 4: havingClause ({@link SqlNode})
    • - *
  • 5: groupClause ({@link SqlNode})
    • + *
  • 4: groupClause ({@link SqlNodeList})
    • + *
  • 5: havingClause ({@link SqlNode})
    • *
  • 6: windowClause ({@link SqlNodeList})
    • - *
  • 7: orderClause ({@link SqlNode})
    • + *
  • 7: qualifyClause ({@link SqlNode})
    • + *
  • 8: orderClause ({@link SqlNodeList})
    • + *
  • 9: offsetClause ({@link SqlNode})
    • + *
  • 10: fetchClause ({@link SqlNode})
    • + *
  • 11: hints ({@link SqlNodeList})
    • + *
  • 12: distinctOn ({@link SqlNodeList})
    • *
*/ public class SqlSelectOperator extends SqlOperator { @@ -80,7 +85,8 @@ private SqlSelectOperator() { (SqlNodeList) operands[8], operands[9], operands[10], - (SqlNodeList) operands[11]); + (SqlNodeList) operands[11], + operands.length > 12 ? (SqlNodeList) operands[12] : null); } /** @@ -116,7 +122,8 @@ public SqlSelect createCall( orderBy, offset, fetch, - hints); + hints, + null); } @Override public void acceptCall( @@ -152,6 +159,13 @@ public SqlSelect createCall( final SqlNode keyword = select.keywordList.get(i); keyword.unparse(writer, 0, 0); } + if (select.isDistinctOn()) { + writer.keyword("ON"); + final SqlWriter.Frame frame = + writer.startList("(", ")"); + castNonNull(select.distinctOn).unparse(writer, 0, 0); + writer.endList(frame); + } writer.topN(select.fetch, select.offset); final SqlNodeList selectClause = select.selectList; writer.list(SqlWriter.FrameTypeEnum.SELECT_LIST, SqlWriter.COMMA, diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlAbstractConformance.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlAbstractConformance.java index 39799a733cad..1a99f52fefb0 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlAbstractConformance.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlAbstractConformance.java @@ -168,4 +168,8 @@ public abstract class SqlAbstractConformance implements SqlConformance { @Override public boolean supportsUnsignedTypes() { return SqlConformanceEnum.DEFAULT.supportsUnsignedTypes(); } + + @Override public boolean isDistinctOnAllowed() { + return SqlConformanceEnum.DEFAULT.isDistinctOnAllowed(); + } } diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlConformance.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlConformance.java index 5e652be9b839..8884328d4b7a 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlConformance.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlConformance.java @@ -668,4 +668,14 @@ enum SelectAliasLookup { * True when the unsigned versions of integer types are supported. */ boolean supportsUnsignedTypes(); + + /** + * Whether {@code SELECT DISTINCT ON} is allowed. + * + *

Among the built-in conformance levels, true in + * {@link SqlConformanceEnum#BABEL}, + * {@link SqlConformanceEnum#LENIENT}; + * false otherwise. + */ + boolean isDistinctOnAllowed(); } diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlConformanceEnum.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlConformanceEnum.java index 8d7d0b530608..196b4c7bec91 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlConformanceEnum.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlConformanceEnum.java @@ -527,4 +527,14 @@ public enum SqlConformanceEnum implements SqlConformance { return false; } } + + @Override public boolean isDistinctOnAllowed() { + switch (this) { + case BABEL: + case LENIENT: + return true; + default: + return false; + } + } } diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlDelegatingConformance.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlDelegatingConformance.java index 00a77b0ee042..645361def848 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlDelegatingConformance.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlDelegatingConformance.java @@ -173,4 +173,8 @@ protected SqlDelegatingConformance(SqlConformance delegate) { @Override public boolean supportsUnsignedTypes() { return delegate.supportsUnsignedTypes(); } + + @Override public boolean isDistinctOnAllowed() { + return delegate.isDistinctOnAllowed(); + } } diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index 6881bae03525..2b58628527ed 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -3069,8 +3069,10 @@ private void registerQuery( if (orderList != null) { // If the query is 'SELECT DISTINCT', restrict the columns // available to the ORDER BY clause. + // DISTINCT ON is an exception: ORDER BY may reference columns + // not in the SELECT list. final SqlValidatorScope selectScope3 = - select.isDistinct() + (select.isDistinct() && !select.isDistinctOn()) ? new AggregatingSelectScope(selectScope, select, true) : selectScope2; OrderByScope orderScope = @@ -4265,6 +4267,7 @@ protected void validateSelect( // dialects you can refer to columns of the select list, e.g. // "SELECT empno AS x FROM emp ORDER BY x" validateOrderList(select); + validateDistinctOnClause(select); if (shouldCheckForRollUp(from)) { checkRollUpInSelectList(select); @@ -4799,6 +4802,54 @@ protected void validateQualifyClause(SqlSelect select) { } } + protected void validateDistinctOnClause(SqlSelect select) { + SqlNodeList distinctOn = select.getDistinctOn(); + if (distinctOn == null || distinctOn.isEmpty()) { + return; + } + + if (!config.conformance().isDistinctOnAllowed()) { + throw newValidationError(select, RESOURCE.distinctOnNotAllowed()); + } + + SqlNodeList orderList = select.getOrderList(); + if (orderList == null || orderList.isEmpty()) { + throw newValidationError(select, RESOURCE.distinctOnRequiresOrderBy()); + } + + if (orderList.size() < distinctOn.size()) { + throw newValidationError(orderList.get(orderList.size() - 1), + RESOURCE.distinctOnOrderByMismatch()); + } + + final SqlValidatorScope orderScope = getOrderScope(select); + for (int i = 0; i < distinctOn.size(); i++) { + SqlNode distinctOnExpr = expand(distinctOn.get(i), orderScope); + SqlNode orderItem = orderList.get(i); + SqlNode orderExpr = stripOrderByModifiers(orderItem); + orderExpr = expand(orderExpr, orderScope); + if (!SqlNode.equalDeep(distinctOnExpr, orderExpr, Litmus.IGNORE)) { + throw newValidationError(orderItem, RESOURCE.distinctOnOrderByMismatch()); + } + } + } + + /** Strips ASC, DESC, NULLS FIRST, NULLS LAST from an ORDER BY item. */ + private static SqlNode stripOrderByModifiers(SqlNode orderExpr) { + while (orderExpr instanceof SqlCall) { + SqlCall call = (SqlCall) orderExpr; + SqlKind kind = call.getKind(); + if (kind == SqlKind.DESCENDING + || kind == SqlKind.NULLS_FIRST + || kind == SqlKind.NULLS_LAST) { + orderExpr = call.operand(0); + } else { + break; + } + } + return orderExpr; + } + protected void validateMustFilterRequirements(SqlSelect select, SelectNamespace ns) { ns.filterRequirement = FilterRequirement.EMPTY; if (select.getFrom() != null) { diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index 4622176a0733..af8f9124a21a 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -720,6 +720,9 @@ private static RelCollation requiredCollation(RelNode r) { if (r instanceof Project) { return requiredCollation(((Project) r).getInput()); } + if (r instanceof Filter) { + return requiredCollation(((Filter) r).getInput()); + } if (r instanceof Delta) { return requiredCollation(((Delta) r).getInput()); } @@ -813,12 +816,18 @@ protected void convertSelectImpl( select.getQualify()); if (select.isDistinct()) { - distinctify(bb, true); + if (!select.isDistinctOn()) { + distinctify(bb, true); + } } - convertOrder( - select, bb, collation, orderExprList, select.getOffset(), - select.getFetch()); + if (select.isDistinctOn()) { + convertOrder(select, bb, collation, orderExprList, null, null); + } else { + convertOrder( + select, bb, collation, orderExprList, select.getOffset(), + select.getFetch()); + } if (select.hasHints()) { final List hints = SqlUtil.getRelHint(hintStrategies, select.getHints()); @@ -839,6 +848,20 @@ protected void convertSelectImpl( } else { bb.setRoot(bb.root(), true); } + + if (select.isDistinctOn()) { + convertDistinctOn(bb, select, collationList); + final @Nullable RexNode offsetExpr = select.getOffset() == null + ? null : convertExpression(select.getOffset()); + final @Nullable RexNode fetchExpr = select.getFetch() == null + ? null : convertExpression(select.getFetch()); + if (offsetExpr != null || fetchExpr != null) { + bb.setRoot( + LogicalSort.create(bb.root(), RelCollations.EMPTY, + offsetExpr, fetchExpr), + false); + } + } } /** @@ -1021,6 +1044,77 @@ private void distinctify( rel.getRowType().getFieldNames(), ImmutableSet.of()), false); } + /** + * Converts a SELECT DISTINCT ON clause into a relational expression + * using ROW_NUMBER() window function. + */ + private void convertDistinctOn(Blackboard bb, SqlSelect select, + List collationList) { + if (bb.root == null) { + throw new IllegalArgumentException("rel must not be null"); + } + final SqlNodeList distinctOn = requireNonNull(select.getDistinctOn(), "distinctOn"); + + relBuilder.push(bb.root()); + final RelDataType inputRowType = bb.root().getRowType(); + + // Build PARTITION BY expressions from DISTINCT ON. + // DISTINCT ON expressions are a prefix of ORDER BY, + // so we can use the field indices from collationList. + final List partitionKeys = new ArrayList<>(); + for (int i = 0; i < distinctOn.size(); i++) { + RelFieldCollation fieldCollation = collationList.get(i); + partitionKeys.add( + rexBuilder.makeInputRef(inputRowType, fieldCollation.getFieldIndex())); + } + + // Build ORDER BY expressions for the window function + final List orderKeys = new ArrayList<>(); + for (RelFieldCollation fieldCollation : collationList) { + RexNode ref = rexBuilder.makeInputRef(inputRowType, fieldCollation.getFieldIndex()); + final Set kinds = new HashSet<>(); + if (fieldCollation.getDirection() == RelFieldCollation.Direction.DESCENDING) { + kinds.add(SqlKind.DESCENDING); + } + switch (fieldCollation.nullDirection) { + case FIRST: + kinds.add(SqlKind.NULLS_FIRST); + break; + case LAST: + kinds.add(SqlKind.NULLS_LAST); + break; + default: + break; + } + orderKeys.add(new RexFieldCollation(ref, kinds)); + } + + final RelDataType bigintType = + typeFactory.createSqlType(SqlTypeName.BIGINT); + final RexNode rowNumber = + rexBuilder.makeOver(bigintType, SqlStdOperatorTable.ROW_NUMBER, ImmutableList.of(), + partitionKeys, ImmutableList.copyOf(orderKeys), + RexWindowBounds.UNBOUNDED_PRECEDING, RexWindowBounds.CURRENT_ROW, + RexWindowExclusion.EXCLUDE_NO_OTHER, true, true, false, false, false); + + // Add ROW_NUMBER as the last column + final List fields = new ArrayList<>(relBuilder.fields()); + fields.add(rowNumber); + relBuilder.project(fields); + + // Filter rn = 1 + relBuilder.filter( + relBuilder.equals( + Util.last(relBuilder.fields()), + relBuilder.literal(BigDecimal.ONE))); + + // Remove the ROW_NUMBER column + relBuilder.project( + Util.skipLast(relBuilder.fields())); + + bb.setRoot(relBuilder.build(), false); + } + /** * Converts a query's ORDER BY clause, if any. * diff --git a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties index ac137d058e70..d374ea908326 100644 --- a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties +++ b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties @@ -170,6 +170,9 @@ FollowingBeforePrecedingError=Upper frame boundary cannot be PRECEDING when lowe WindowNameMustBeSimple=Window name must be a simple identifier DuplicateWindowName=Duplicate window names not allowed EmptyWindowSpec=Empty window specification not allowed +DistinctOnNotAllowed=SELECT DISTINCT ON is not supported under the current SQL conformance level +DistinctOnRequiresOrderBy=SELECT DISTINCT ON requires an ORDER BY clause +DistinctOnOrderByMismatch=SELECT DISTINCT ON expressions must match initial ORDER BY expressions DupWindowSpec=Duplicate window specification not allowed in the same window clause QualifyExpressionMustContainWindowFunction=QUALIFY expression ''{0}'' must contain a window function RankWithFrame=ROW/RANGE not allowed with RANK, DENSE_RANK, ROW_NUMBER, PERCENTILE_CONT/DISC or LAG/LEAD functions diff --git a/core/src/test/codegen/config.fmpp b/core/src/test/codegen/config.fmpp index 99ecb93da962..3f985001913a 100644 --- a/core/src/test/codegen/config.fmpp +++ b/core/src/test/codegen/config.fmpp @@ -69,6 +69,7 @@ data: { implementationFiles: [ "parserImpls.ftl" ] + includeDistinctOn: true } } diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java index dc18d05520ea..5bbea360288f 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java @@ -6039,4 +6039,35 @@ void checkUserDefinedOrderByOver(NullCollation nullCollation) { final String sql = "select distinct deptno, deptno, empno, 1, 'a' from emp order by rand(), 1"; sql(sql).ok(); } + + /** Test case of + * [CALCITE-5406] + * Support the SELECT DISTINCT ON statement for PostgreSQL dialect. */ + @Test void testDistinctOnSimple() { + final String sql = "SELECT DISTINCT ON (deptno) empno, ename\n" + + "FROM emp\n" + + "ORDER BY deptno, empno"; + sql(sql).withConformance(SqlConformanceEnum.LENIENT).ok(); + } + + /** Test case of + * [CALCITE-5406] + * Support the SELECT DISTINCT ON statement for PostgreSQL dialect. */ + @Test void testDistinctOnMultiple() { + final String sql = "SELECT DISTINCT ON (deptno, job) empno, ename\n" + + "FROM emp\n" + + "ORDER BY deptno, job, hiredate DESC"; + sql(sql).withConformance(SqlConformanceEnum.LENIENT).ok(); + } + + /** Test case of + * [CALCITE-5406] + * Support the SELECT DISTINCT ON statement for PostgreSQL dialect. */ + @Test void testDistinctOnWithLimit() { + final String sql = "SELECT DISTINCT ON (deptno) empno, ename\n" + + "FROM emp\n" + + "ORDER BY deptno, empno\n" + + "LIMIT 5"; + sql(sql).withConformance(SqlConformanceEnum.LENIENT).ok(); + } } diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 3d7eda0edf55..a0d7480b73a5 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -6507,7 +6507,122 @@ void testReturnsCorrectRowTypeOnCombinedJoin() { f.withSql(qualifyOnMultipleWindowFunctions).ok(); } - /** Negative tests for the {@code QUALIFY} clause. */ + /** Test case of + * [CALCITE-5406] + * Support the SELECT DISTINCT ON statement for PostgreSQL dialect. */ + @Test void testDistinctOnNotAllowed() { + // DISTINCT ON is not allowed under default SQL conformance + sql("^SELECT DISTINCT ON (deptno) empno FROM emp^ ORDER BY deptno") + .fails("SELECT DISTINCT ON is not supported under the current SQL conformance level"); + } + + /** Test case of + * [CALCITE-5406] + * Support the SELECT DISTINCT ON statement for PostgreSQL dialect. */ + @Test void testDistinctOnPositive() { + final SqlValidatorFixture f = + fixture().withConformance(SqlConformanceEnum.LENIENT); + + f.withSql("SELECT DISTINCT ON (deptno) empno, ename FROM emp ORDER BY deptno, empno") + .ok(); + + f.withSql("SELECT DISTINCT ON (deptno, job) empno FROM emp ORDER BY deptno, job, hiredate") + .ok(); + + f.withSql("SELECT DISTINCT ON (deptno) empno FROM emp ORDER BY deptno DESC") + .ok(); + + f.withSql("SELECT DISTINCT ON (deptno) empno FROM emp ORDER BY deptno NULLS FIRST") + .ok(); + } + + /** Test case of + * [CALCITE-5406] + * Support the SELECT DISTINCT ON statement for PostgreSQL dialect. */ + @Test void testDistinctOnNegative() { + final SqlValidatorFixture f = + fixture().withConformance(SqlConformanceEnum.LENIENT); + + // DISTINCT ON requires ORDER BY + f.withSql("^SELECT DISTINCT ON (deptno) empno FROM emp^") + .fails("SELECT DISTINCT ON requires an ORDER BY clause"); + + // ORDER BY must contain all DISTINCT ON expressions as prefix + f.withSql("SELECT DISTINCT ON (deptno, job) empno FROM emp ORDER BY ^deptno^") + .fails("SELECT DISTINCT ON expressions must match initial ORDER BY expressions"); + + // ORDER BY prefix must match exactly + f.withSql("SELECT DISTINCT ON (deptno, job) empno FROM emp ORDER BY ^job^, deptno") + .fails("SELECT DISTINCT ON expressions must match initial ORDER BY expressions"); + + // DISTINCT ON with extra ORDER BY is ok + f.withSql("SELECT DISTINCT ON (deptno) empno FROM emp ORDER BY deptno, job") + .ok(); + + // Duplicate expressions in DISTINCT ON are allowed but must be matched in ORDER BY + f.withSql("SELECT DISTINCT ON (deptno, deptno) deptno, empno FROM emp ORDER BY deptno, deptno") + .ok(); + + // DISTINCT ON with expression + f.withSql("SELECT DISTINCT ON (empno % 2) empno, ename FROM emp ORDER BY empno % 2") + .ok(); + + // Empty DISTINCT ON is not allowed (parse error) + f.withSql("SELECT DISTINCT ON ((^)^) deptno FROM emp") + .fails("(?s)Encountered \"\\)\" at .*"); + + // DISTINCT ON can reference alias (like ORDER BY) + f.withSql("SELECT DISTINCT ON (x) empno AS x, deptno FROM emp ORDER BY x") + .ok(); + + // DISTINCT ON with alias-column name clash: alias in SELECT takes precedence + f.withSql("SELECT DISTINCT ON (deptno) empno AS deptno, deptno AS d FROM emp ORDER BY deptno") + .ok(); + + // DISTINCT ON with qualified column reference + f.withSql("SELECT DISTINCT ON (e.deptno) e.deptno AS x, e.empno " + + "FROM emp AS e ORDER BY e.deptno") + .ok(); + + // Integer literal in both DISTINCT ON and ORDER BY matches at validator + // (ordinal resolution happens later in SqlToRelConverter) + f.withSql("SELECT DISTINCT ON (2) empno, ename FROM emp ORDER BY 2") + .ok(); + + // Expressions in DISTINCT ON (not ordinals) + f.withSql("SELECT DISTINCT ON (empno % 2, CHAR_LENGTH(ename)) empno, ename " + + "FROM emp ORDER BY empno % 2, CHAR_LENGTH(ename)") + .ok(); + + // DISTINCT ON referencing non-projected column requires ORDER BY + f.withSql("^SELECT DISTINCT ON (deptno) empno, ename FROM emp^") + .fails("SELECT DISTINCT ON requires an ORDER BY clause"); + + // DISTINCT ON with aggregate query + f.withSql("SELECT DISTINCT ON (deptno) deptno, SUM(sal) " + + "FROM emp GROUP BY deptno ORDER BY deptno") + .ok(); + + // DISTINCT ON with aggregate expression + f.withSql("SELECT DISTINCT ON (SUM(sal)) deptno, SUM(sal) " + + "FROM emp GROUP BY deptno ORDER BY SUM(sal)") + .ok(); + + // DISTINCT ON with aggregate query and alias + f.withSql("SELECT DISTINCT ON (sum_sal) deptno, SUM(sal) AS sum_sal " + + "FROM emp GROUP BY deptno ORDER BY sum_sal") + .ok(); + + // DISTINCT ON with USING join (requires qualified reference due to Calcite limitation) + f.withSql("SELECT DISTINCT ON (emp.deptno) * " + + "FROM emp JOIN dept USING (deptno) ORDER BY emp.deptno") + .ok(); + + // DISTINCT ON with NATURAL join (requires qualified reference due to Calcite limitation) + f.withSql("SELECT DISTINCT ON (emp.deptno) * FROM emp NATURAL JOIN dept ORDER BY emp.deptno") + .ok(); + } + @Test void testQualifyNegative() { final SqlValidatorFixture f = fixture().withConformance(SqlConformanceEnum.LENIENT); diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index a4dc246f8e25..4a645a5e51b4 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -1887,6 +1887,62 @@ LogicalTableModify(table=[[CATALOG, SALES, EMP]], operation=[DELETE], flattened= LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], SLACKER=[$8]) LogicalFilter(condition=[=($7, 10)]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) +]]> + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/server/src/main/codegen/config.fmpp b/server/src/main/codegen/config.fmpp index e5c3eaf4d27a..ba64f0b71a51 100644 --- a/server/src/main/codegen/config.fmpp +++ b/server/src/main/codegen/config.fmpp @@ -99,6 +99,7 @@ data: { implementationFiles: [ "parserImpls.ftl" ] + includeDistinctOn: true } } diff --git a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java index fad96f70b956..7c0fde601a78 100644 --- a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java +++ b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java @@ -6081,6 +6081,104 @@ private static Matcher isCharLiteral(String s) { sql(sql).fails("(?s).*Encountered \"QUALIFY\" at .*"); } + /** Test case of + * [CALCITE-5406] + * Support the SELECT DISTINCT ON statement for PostgreSQL dialect. */ + @Test void testDistinctOn() { + final String sql = "SELECT DISTINCT ON (deptno) empno, ename\n" + + "FROM emp\n" + + "ORDER BY deptno, empno"; + + final String expected = "SELECT DISTINCT ON (`DEPTNO`) `EMPNO`, `ENAME`\n" + + "FROM `EMP`\n" + + "ORDER BY `DEPTNO`, `EMPNO`"; + sql(sql).ok(expected); + } + + /** Test case of + * [CALCITE-5406] + * Support the SELECT DISTINCT ON statement for PostgreSQL dialect. */ + @Test void testDistinctOnMultiple() { + final String sql = "SELECT DISTINCT ON (deptno, job) empno, ename\n" + + "FROM emp\n" + + "ORDER BY deptno, job, hiredate DESC"; + + final String expected = "SELECT DISTINCT ON (`DEPTNO`, `JOB`) `EMPNO`, `ENAME`\n" + + "FROM `EMP`\n" + + "ORDER BY `DEPTNO`, `JOB`, `HIREDATE` DESC"; + sql(sql).ok(expected); + } + + /** Test case of + * [CALCITE-5406] + * Support the SELECT DISTINCT ON statement for PostgreSQL dialect. */ + @Test void testDistinctOnWithEverything() { + final String sql = "SELECT DISTINCT ON (deptno) empno, ename\n" + + "FROM emp\n" + + "WHERE deptno > 3\n" + + "GROUP BY deptno, empno, ename\n" + + "HAVING COUNT(*) > 1\n" + + "ORDER BY deptno, empno\n" + + "LIMIT 5\n"; + + final String expected = "SELECT DISTINCT ON (`DEPTNO`) `EMPNO`, `ENAME`\n" + + "FROM `EMP`\n" + + "WHERE (`DEPTNO` > 3)\n" + + "GROUP BY `DEPTNO`, `EMPNO`, `ENAME`\n" + + "HAVING (COUNT(*) > 1)\n" + + "ORDER BY `DEPTNO`, `EMPNO`\n" + + "FETCH NEXT 5 ROWS ONLY"; + sql(sql).ok(expected); + } + + /** Test case of + * [CALCITE-5406] + * Support the SELECT DISTINCT ON statement for PostgreSQL dialect. */ + @Test void testDistinctOnWithOffset() { + final String sql = "SELECT DISTINCT ON (deptno) empno, ename\n" + + "FROM emp\n" + + "ORDER BY deptno, empno\n" + + "OFFSET 10 ROWS\n" + + "FETCH NEXT 5 ROWS ONLY"; + + final String expected = "SELECT DISTINCT ON (`DEPTNO`) `EMPNO`, `ENAME`\n" + + "FROM `EMP`\n" + + "ORDER BY `DEPTNO`, `EMPNO`\n" + + "OFFSET 10 ROWS\n" + + "FETCH NEXT 5 ROWS ONLY"; + sql(sql).ok(expected); + } + + /** Test case of + * [CALCITE-5406] + * Support the SELECT DISTINCT ON statement for PostgreSQL dialect. */ + @Test void testDistinctOnWithExpression() { + final String sql = "SELECT DISTINCT ON (deptno + 1) empno, ename\n" + + "FROM emp\n" + + "ORDER BY deptno + 1, empno"; + + final String expected = "SELECT DISTINCT ON ((`DEPTNO` + 1)) `EMPNO`, `ENAME`\n" + + "FROM `EMP`\n" + + "ORDER BY (`DEPTNO` + 1), `EMPNO`"; + sql(sql).ok(expected); + } + + /** Test case of + * [CALCITE-5406] + * Support the SELECT DISTINCT ON statement for PostgreSQL dialect. */ + @Test void testDistinctOnWithJoin() { + final String sql = "SELECT DISTINCT ON (e.deptno) e.empno, d.dname\n" + + "FROM emp AS e JOIN dept AS d ON e.deptno = d.deptno\n" + + "ORDER BY e.deptno, e.empno"; + + final String expected = "SELECT DISTINCT ON (`E`.`DEPTNO`) " + + "`E`.`EMPNO`, `D`.`DNAME`\n" + + "FROM `EMP` AS `E`\n" + + "INNER JOIN `DEPT` AS `D` ON (`E`.`DEPTNO` = `D`.`DEPTNO`)\n" + + "ORDER BY `E`.`DEPTNO`, `E`.`EMPNO`"; + sql(sql).ok(expected); + } + @Test void testNullTreatment() { sql("select lead(x) respect nulls over (w) from t") .ok("SELECT (LEAD(`X`) RESPECT NULLS OVER (`W`))\n"