amaliujia commented on a change in pull request #2109: URL: https://github.com/apache/calcite/pull/2109#discussion_r470268938
########## File path: core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSortRule.java ########## @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.adapter.enumerable; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelRule; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Sort; +import org.apache.calcite.rel.logical.LogicalSort; + +/** + * Rule to convert an {@link EnumerableLimit} of on + * {@link EnumerableSort} into an {@link EnumerableLimitSort}. + */ +public class EnumerableLimitSortRule extends RelRule<EnumerableLimitSortRule.Config> { + + /** + * Creates a EnumerableLimitSortRule. + */ + public EnumerableLimitSortRule(Config config) { + super(config); + } + + @Override public void onMatch(RelOptRuleCall call) { + final LogicalSort sort = call.rel(0); + RelNode input = sort.getInput(); + final Sort o = EnumerableLimitSort.create(// Review comment: remove the extra `//` ########## File path: core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java ########## @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.adapter.enumerable; + +import org.apache.calcite.linq4j.tree.BlockBuilder; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptCost; +import org.apache.calcite.plan.RelOptPlanner; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.RelCollation; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Sort; +import org.apache.calcite.rel.metadata.RelMetadataQuery; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.util.BuiltInMethod; +import org.apache.calcite.util.Pair; + +import static org.apache.calcite.adapter.enumerable.EnumerableLimit.getExpression; + +/** + * Implementation of {@link org.apache.calcite.rel.core.Sort} in + * {@link org.apache.calcite.adapter.enumerable.EnumerableConvention enumerable calling convention}. + * It optimizes sorts that have a limit and an optional offset. + */ +public class EnumerableLimitSort extends Sort implements EnumerableRel { + + /** + * Creates an EnumerableLimitSort. + * + * <p>Use {@link #create} unless you know what you're doing. + */ + public EnumerableLimitSort( + RelOptCluster cluster, + RelTraitSet traitSet, + RelNode input, + RelCollation collation, + RexNode offset, + RexNode fetch) { + super(cluster, traitSet, input, collation, offset, fetch); + assert this.getConvention() instanceof EnumerableConvention; + assert this.getConvention() == input.getConvention(); + } + + /** Creates an EnumerableLimitSort. */ + public static EnumerableLimitSort create( + RelNode input, + RelCollation collation, + RexNode offset, + RexNode fetch) { + final RelOptCluster cluster = input.getCluster(); + final RelTraitSet traitSet = cluster.traitSetOf(EnumerableConvention.INSTANCE).replace( + collation); + return new EnumerableLimitSort(cluster, traitSet, input, collation, offset, fetch); + } + + @Override public EnumerableLimitSort copy( + RelTraitSet traitSet, + RelNode newInput, + RelCollation newCollation, + RexNode offset, + RexNode fetch) { + return new EnumerableLimitSort( + this.getCluster(), + traitSet, + newInput, + newCollation, + offset, + fetch); + } + + @Override public Result implement(EnumerableRelImplementor implementor, Prefer pref) { + final BlockBuilder builder = new BlockBuilder(); + final EnumerableRel child = (EnumerableRel) this.getInput(); + final Result result = implementor.visitChild(this, 0, child, pref); + final PhysType physType = PhysTypeImpl.of( + implementor.getTypeFactory(), + this.getRowType(), + result.format); + Expression childExp = builder.append("child", result.block); + + PhysType inputPhysType = result.physType; + final Pair<Expression, Expression> pair = + inputPhysType.generateCollationKey(this.collation.getFieldCollations()); + + Expression fetchVal; + if (this.fetch == null) { + fetchVal = Expressions.constant(Integer.valueOf(Integer.MAX_VALUE)); + } else { + fetchVal = getExpression(this.fetch); + } + + Expression offsetVal = this.offset == null ? Expressions.constant(Integer.valueOf(0)) + : getExpression(this.offset); + + builder.add( + Expressions.return_( + null, Expressions.call( + BuiltInMethod.ORDER_BY_WITH_FETCH_AND_OFFSET.method, Expressions.list( + childExp, + builder.append("keySelector", pair.left)) + .appendIfNotNull(builder.appendIfNotNull("comparator", pair.right)) + .appendIfNotNull( + builder.appendIfNotNull("offset", + Expressions.constant(offsetVal))) + .appendIfNotNull( + builder.appendIfNotNull("fetch", + Expressions.constant(fetchVal))) + ))); + return implementor.result(physType, builder.toBlock()); + } + + @Override public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { + final double rowCount = mq.getRowCount(this.input).doubleValue(); + double toSort = this.fetch == null ? rowCount : RexLiteral.intValue(this.fetch); + if (this.offset != null) { + toSort += RexLiteral.intValue(this.offset); + } + toSort = Math.min(rowCount, toSort); Review comment: Why not use `max`? ########## File path: linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java ########## @@ -2624,6 +2624,86 @@ public static boolean isMergeJoinSupported(JoinType joinType) { }; } + + /** + * A sort implementation optimized for a sort with a fetch size (LIMIT). + * @param offset how many rows are skipped from the sorted output. + * Must be greater than or equal to 0. + * @param fetch how many rows are retrieved. Must be greater than 0. + */ + public static <TSource, TKey> Enumerable<TSource> orderBy( Review comment: Does it make sense to name this function as `topNOrderBy` or `topNSortLimit`? ########## File path: core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java ########## @@ -1039,6 +1039,54 @@ private void basePushFilterPastAggWithGroupingSets(boolean unchanged) { .check(); } + /** + * Test if limit and sort are replaced by a limit sort. + * Test case for + * <a href="https://issues.apache.org/jira/browse/CALCITE-3920";>[CALCITE-3920] + * Improve ORDER BY computation in Enumerable convention by exploiting LIMIT</a>. + */ + @Test void testLimitSort() { + final String sql = "select mgr from sales.emp\n" + + "union select mgr from sales.emp\n" + + "order by mgr limit 10 offset 5"; + + VolcanoPlanner planner = new VolcanoPlanner(null, null); + planner.addRelTraitDef(ConventionTraitDef.INSTANCE); + RelOptUtil.registerDefaultRules(planner, false, false); + planner.addRule(EnumerableRules.ENUMERABLE_LIMIT_SORT_RULE); + + Tester tester = createTester().withDecorrelation(true) + .withClusterFactory( + relOptCluster -> RelOptCluster.create(planner, relOptCluster.getRexBuilder())); + + RelRoot root = tester.convertSqlToRel(sql); + + String planBefore = NL + RelOptUtil.toString(root.rel); + getDiffRepos().assertEquals("planBefore", "${planBefore}", planBefore); + + RuleSet ruleSet = + RuleSets.ofList( + EnumerableRules.ENUMERABLE_SORT_RULE, + EnumerableRules.ENUMERABLE_LIMIT_RULE, + EnumerableRules.ENUMERABLE_LIMIT_SORT_RULE, Review comment: Do you need ``` EnumerableRules.ENUMERABLE_SORT_RULE, EnumerableRules.ENUMERABLE_LIMIT_RULE, ``` as you have listed `EnumerableRules.ENUMERABLE_LIMIT_SORT_RULE` in the list? ########## File path: core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableLimitSort.java ########## @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.adapter.enumerable; + +import org.apache.calcite.linq4j.tree.BlockBuilder; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptCost; +import org.apache.calcite.plan.RelOptPlanner; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.RelCollation; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Sort; +import org.apache.calcite.rel.metadata.RelMetadataQuery; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.util.BuiltInMethod; +import org.apache.calcite.util.Pair; + +import static org.apache.calcite.adapter.enumerable.EnumerableLimit.getExpression; + +/** + * Implementation of {@link org.apache.calcite.rel.core.Sort} in + * {@link org.apache.calcite.adapter.enumerable.EnumerableConvention enumerable calling convention}. + * It optimizes sorts that have a limit and an optional offset. + */ +public class EnumerableLimitSort extends Sort implements EnumerableRel { Review comment: Out of curiosity: why do you decide to extend `Sort`? ########## File path: linq4j/src/main/java/org/apache/calcite/linq4j/EnumerableDefaults.java ########## @@ -2624,6 +2624,86 @@ public static boolean isMergeJoinSupported(JoinType joinType) { }; } + + /** + * A sort implementation optimized for a sort with a fetch size (LIMIT). + * @param offset how many rows are skipped from the sorted output. + * Must be greater than or equal to 0. + * @param fetch how many rows are retrieved. Must be greater than 0. + */ + public static <TSource, TKey> Enumerable<TSource> orderBy( + Enumerable<TSource> source, + Function1<TSource, TKey> keySelector, + Comparator<TKey> comparator, + int offset, int fetch) { + return new AbstractEnumerable<TSource>() { + @Override public Enumerator<TSource> enumerator() { + TreeMap<TKey, List<TSource>> map = new TreeMap<>(comparator); + long size = 0; + long needed = fetch + offset; + + try (Enumerator<TSource> os = source.enumerator()) { + while (os.moveNext()) { + TSource o = os.current(); + TKey key = keySelector.apply(o); + if (needed >= 0 && size >= needed) { + if (comparator.compare(key, map.lastKey()) >= 0) { + continue; + } + // remove last entry from tree map + List<TSource> l = map.get(map.lastKey()); + if (l.size() == 1) { + map.remove(map.lastKey()); + } else { + l.remove(l.size() - 1); + } + size--; + } + map.compute(key, (k, l) -> { + if (l == null) { + return Collections.singletonList(o); Review comment: +1. I was also looking for some comments to explain the algorithm briefly. ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected]
