[GitHub] [calcite] laurentgo commented on a change in pull request #2016: [CALCITE-3786] Add Digest interface to enable efficient hashCode(equa…
laurentgo commented on a change in pull request #2016:
URL: https://github.com/apache/calcite/pull/2016#discussion_r439694355
##
File path: core/src/main/java/org/apache/calcite/plan/Digest.java
##
@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.plan;
+
+import org.apache.calcite.plan.hep.HepRelVertex;
+import org.apache.calcite.plan.volcano.RelSubset;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.hint.Hintable;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.util.Pair;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * A short description of relational expression's type, inputs, and
+ * other properties. The digest uniquely identifies the node; another node
+ * is equivalent if and only if it has the same value.
+ *
+ * Row type is part of the digest for the rare occasion that similar
+ * expressions have different types, e.g. variants of
+ * {@code Project(child=rel#1, a=null)} where a is a null INTEGER or a
+ * null VARCHAR(10). Row type is represented as fieldTypes only, so {@code
RelNode}
+ * that differ with field names only are treated equal.
+ * For instance, {@code Project(input=rel#1,empid=$0)} and {@code
Project(input=rel#1,deptno=$0)}
+ * are equal.
+ *
+ * Computed by {@code org.apache.calcite.rel.AbstractRelNode#computeDigest},
+ * assigned by {@link org.apache.calcite.rel.AbstractRelNode#onRegister},
+ * returned by {@link org.apache.calcite.rel.AbstractRelNode#getDigest()}.
+ */
+public class Digest implements Comparable {
+
+ //~ Instance fields
+
+ final int hashCode;
+ final List> items;
+ private final RelNode rel;
+
+ // Used for debugging, computed lazily.
+ private String digest = null;
+
+ //~ Constructors ---
+
+ /**
+ * Creates a digest with given rel and properties.
+ *
+ * @param rel The rel
+ * @param items The properties, e.g. the inputs, the type, the traits and so
on
+ */
+ private Digest(RelNode rel, List> items) {
+this.rel = rel;
+this.items = normalizeContents(items);
+this.hashCode = computeIdentity(rel, this.items);
+ }
+
+ /**
+ * Creates a digest with given rel, the digest is computed as simple,
+ * see {@link #simpleRelDigest(RelNode)}.
+ */
+ private Digest(RelNode rel) {
+this(rel, simpleRelDigest(rel));
+ }
+
+ /** Creates a digest with given rel and string format digest. */
+ private Digest(RelNode rel, String digest) {
+this.rel = rel;
+this.items = Collections.emptyList();
+this.digest = digest;
+this.hashCode = this.digest.hashCode();
+ }
+
+ /** Returns the identity of this digest which is used to speedup hashCode
and equals. */
+ private static int computeIdentity(RelNode rel, List>
contents) {
+return Objects.hash(collect(rel, contents, false));
+ }
+
+ /**
+ * Collects the items used for {@link #hashCode} and {@link #equals}.
+ *
+ * Generally, the items used for hashCode and equals should be the same.
The exception
+ * is the row type of the relational expression: the row type is needed
because during
+ * planning, new equivalent rels may be produced with changed fields
nullability
+ * (i.e. most of them comes from the rex simplify or constant reduction).
+ * This expects to be rare case, so the hashcode is computed without row type
+ * but when it conflicts, we compare with the row type involved(sans field
names).
+ *
+ * @param rel The rel to compute digest
+ * @param contents The rel properties should be considered in digest
+ * @param withType Whether to involve the row type
+ */
+ private static Object[] collect(
+ RelNode rel,
+ List> contents,
+ boolean withType) {
+List hashCodeItems = new ArrayList<>();
+// The type name.
+hashCodeItems.add(rel.getRelTypeName());
+// The traits.
+hashCodeItems.addAll(rel.getTraitSet());
+// The hints.
+if (rel instanceof Hintable) {
+ hashCodeItems.addAll(((Hintable) rel).getHints());
+}
+if
[GitHub] [calcite] laurentgo commented on a change in pull request #2016: [CALCITE-3786] Add Digest interface to enable efficient hashCode(equa…
laurentgo commented on a change in pull request #2016:
URL: https://github.com/apache/calcite/pull/2016#discussion_r439693715
##
File path: core/src/main/java/org/apache/calcite/plan/Digest.java
##
@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.plan;
+
+import org.apache.calcite.plan.hep.HepRelVertex;
+import org.apache.calcite.plan.volcano.RelSubset;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.hint.Hintable;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.util.Pair;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * A short description of relational expression's type, inputs, and
+ * other properties. The digest uniquely identifies the node; another node
+ * is equivalent if and only if it has the same value.
+ *
+ * Row type is part of the digest for the rare occasion that similar
+ * expressions have different types, e.g. variants of
+ * {@code Project(child=rel#1, a=null)} where a is a null INTEGER or a
+ * null VARCHAR(10). Row type is represented as fieldTypes only, so {@code
RelNode}
+ * that differ with field names only are treated equal.
+ * For instance, {@code Project(input=rel#1,empid=$0)} and {@code
Project(input=rel#1,deptno=$0)}
+ * are equal.
+ *
+ * Computed by {@code org.apache.calcite.rel.AbstractRelNode#computeDigest},
+ * assigned by {@link org.apache.calcite.rel.AbstractRelNode#onRegister},
+ * returned by {@link org.apache.calcite.rel.AbstractRelNode#getDigest()}.
+ */
+public class Digest implements Comparable {
+
+ //~ Instance fields
+
+ final int hashCode;
+ final List> items;
+ private final RelNode rel;
+
+ // Used for debugging, computed lazily.
+ private String digest = null;
+
+ //~ Constructors ---
+
+ /**
+ * Creates a digest with given rel and properties.
+ *
+ * @param rel The rel
+ * @param items The properties, e.g. the inputs, the type, the traits and so
on
+ */
+ private Digest(RelNode rel, List> items) {
+this.rel = rel;
+this.items = normalizeContents(items);
+this.hashCode = computeIdentity(rel, this.items);
+ }
+
+ /**
+ * Creates a digest with given rel, the digest is computed as simple,
+ * see {@link #simpleRelDigest(RelNode)}.
+ */
+ private Digest(RelNode rel) {
+this(rel, simpleRelDigest(rel));
+ }
+
+ /** Creates a digest with given rel and string format digest. */
+ private Digest(RelNode rel, String digest) {
+this.rel = rel;
+this.items = Collections.emptyList();
+this.digest = digest;
+this.hashCode = this.digest.hashCode();
+ }
+
+ /** Returns the identity of this digest which is used to speedup hashCode
and equals. */
+ private static int computeIdentity(RelNode rel, List>
contents) {
+return Objects.hash(collect(rel, contents, false));
+ }
+
+ /**
+ * Collects the items used for {@link #hashCode} and {@link #equals}.
+ *
+ * Generally, the items used for hashCode and equals should be the same.
The exception
+ * is the row type of the relational expression: the row type is needed
because during
+ * planning, new equivalent rels may be produced with changed fields
nullability
+ * (i.e. most of them comes from the rex simplify or constant reduction).
+ * This expects to be rare case, so the hashcode is computed without row type
+ * but when it conflicts, we compare with the row type involved(sans field
names).
+ *
+ * @param rel The rel to compute digest
+ * @param contents The rel properties should be considered in digest
+ * @param withType Whether to involve the row type
+ */
+ private static Object[] collect(
+ RelNode rel,
+ List> contents,
+ boolean withType) {
+List hashCodeItems = new ArrayList<>();
+// The type name.
+hashCodeItems.add(rel.getRelTypeName());
+// The traits.
+hashCodeItems.addAll(rel.getTraitSet());
+// The hints.
+if (rel instanceof Hintable) {
+ hashCodeItems.addAll(((Hintable) rel).getHints());
+}
+if
[GitHub] [calcite] danny0405 commented on a change in pull request #2016: [CALCITE-3786] Add Digest interface to enable efficient hashCode(equa…
danny0405 commented on a change in pull request #2016:
URL: https://github.com/apache/calcite/pull/2016#discussion_r439693708
##
File path: core/src/main/java/org/apache/calcite/plan/Digest.java
##
@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.plan;
+
+import org.apache.calcite.plan.hep.HepRelVertex;
+import org.apache.calcite.plan.volcano.RelSubset;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.hint.Hintable;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.util.Pair;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * A short description of relational expression's type, inputs, and
+ * other properties. The digest uniquely identifies the node; another node
+ * is equivalent if and only if it has the same value.
+ *
+ * Row type is part of the digest for the rare occasion that similar
+ * expressions have different types, e.g. variants of
+ * {@code Project(child=rel#1, a=null)} where a is a null INTEGER or a
+ * null VARCHAR(10). Row type is represented as fieldTypes only, so {@code
RelNode}
+ * that differ with field names only are treated equal.
+ * For instance, {@code Project(input=rel#1,empid=$0)} and {@code
Project(input=rel#1,deptno=$0)}
+ * are equal.
+ *
+ * Computed by {@code org.apache.calcite.rel.AbstractRelNode#computeDigest},
+ * assigned by {@link org.apache.calcite.rel.AbstractRelNode#onRegister},
+ * returned by {@link org.apache.calcite.rel.AbstractRelNode#getDigest()}.
+ */
+public class Digest implements Comparable {
+
+ //~ Instance fields
+
+ final int hashCode;
+ final List> items;
+ private final RelNode rel;
+
+ // Used for debugging, computed lazily.
+ private String digest = null;
+
+ //~ Constructors ---
+
+ /**
+ * Creates a digest with given rel and properties.
+ *
+ * @param rel The rel
+ * @param items The properties, e.g. the inputs, the type, the traits and so
on
+ */
+ private Digest(RelNode rel, List> items) {
+this.rel = rel;
+this.items = normalizeContents(items);
+this.hashCode = computeIdentity(rel, this.items);
+ }
+
+ /**
+ * Creates a digest with given rel, the digest is computed as simple,
+ * see {@link #simpleRelDigest(RelNode)}.
+ */
+ private Digest(RelNode rel) {
+this(rel, simpleRelDigest(rel));
+ }
+
+ /** Creates a digest with given rel and string format digest. */
+ private Digest(RelNode rel, String digest) {
+this.rel = rel;
+this.items = Collections.emptyList();
+this.digest = digest;
+this.hashCode = this.digest.hashCode();
+ }
+
+ /** Returns the identity of this digest which is used to speedup hashCode
and equals. */
+ private static int computeIdentity(RelNode rel, List>
contents) {
+return Objects.hash(collect(rel, contents, false));
+ }
+
+ /**
+ * Collects the items used for {@link #hashCode} and {@link #equals}.
+ *
+ * Generally, the items used for hashCode and equals should be the same.
The exception
+ * is the row type of the relational expression: the row type is needed
because during
+ * planning, new equivalent rels may be produced with changed fields
nullability
+ * (i.e. most of them comes from the rex simplify or constant reduction).
+ * This expects to be rare case, so the hashcode is computed without row type
+ * but when it conflicts, we compare with the row type involved(sans field
names).
+ *
+ * @param rel The rel to compute digest
+ * @param contents The rel properties should be considered in digest
+ * @param withType Whether to involve the row type
+ */
+ private static Object[] collect(
+ RelNode rel,
+ List> contents,
+ boolean withType) {
+List hashCodeItems = new ArrayList<>();
+// The type name.
+hashCodeItems.add(rel.getRelTypeName());
+// The traits.
+hashCodeItems.addAll(rel.getTraitSet());
+// The hints.
+if (rel instanceof Hintable) {
+ hashCodeItems.addAll(((Hintable) rel).getHints());
+}
+if
[GitHub] [calcite] laurentgo commented on a change in pull request #2016: [CALCITE-3786] Add Digest interface to enable efficient hashCode(equa…
laurentgo commented on a change in pull request #2016:
URL: https://github.com/apache/calcite/pull/2016#discussion_r439693614
##
File path: core/src/main/java/org/apache/calcite/plan/Digest.java
##
@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.plan;
+
+import org.apache.calcite.plan.hep.HepRelVertex;
+import org.apache.calcite.plan.volcano.RelSubset;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.hint.Hintable;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.util.Pair;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * A short description of relational expression's type, inputs, and
+ * other properties. The digest uniquely identifies the node; another node
+ * is equivalent if and only if it has the same value.
+ *
+ * Row type is part of the digest for the rare occasion that similar
+ * expressions have different types, e.g. variants of
+ * {@code Project(child=rel#1, a=null)} where a is a null INTEGER or a
+ * null VARCHAR(10). Row type is represented as fieldTypes only, so {@code
RelNode}
+ * that differ with field names only are treated equal.
+ * For instance, {@code Project(input=rel#1,empid=$0)} and {@code
Project(input=rel#1,deptno=$0)}
+ * are equal.
+ *
+ * Computed by {@code org.apache.calcite.rel.AbstractRelNode#computeDigest},
+ * assigned by {@link org.apache.calcite.rel.AbstractRelNode#onRegister},
+ * returned by {@link org.apache.calcite.rel.AbstractRelNode#getDigest()}.
+ */
+public class Digest implements Comparable {
+
+ //~ Instance fields
+
+ final int hashCode;
+ final List> items;
+ private final RelNode rel;
+
+ // Used for debugging, computed lazily.
+ private String digest = null;
+
+ //~ Constructors ---
+
+ /**
+ * Creates a digest with given rel and properties.
+ *
+ * @param rel The rel
+ * @param items The properties, e.g. the inputs, the type, the traits and so
on
+ */
+ private Digest(RelNode rel, List> items) {
+this.rel = rel;
+this.items = normalizeContents(items);
+this.hashCode = computeIdentity(rel, this.items);
+ }
+
+ /**
+ * Creates a digest with given rel, the digest is computed as simple,
+ * see {@link #simpleRelDigest(RelNode)}.
+ */
+ private Digest(RelNode rel) {
+this(rel, simpleRelDigest(rel));
+ }
+
+ /** Creates a digest with given rel and string format digest. */
+ private Digest(RelNode rel, String digest) {
+this.rel = rel;
+this.items = Collections.emptyList();
+this.digest = digest;
+this.hashCode = this.digest.hashCode();
+ }
+
+ /** Returns the identity of this digest which is used to speedup hashCode
and equals. */
+ private static int computeIdentity(RelNode rel, List>
contents) {
+return Objects.hash(collect(rel, contents, false));
+ }
+
+ /**
+ * Collects the items used for {@link #hashCode} and {@link #equals}.
+ *
+ * Generally, the items used for hashCode and equals should be the same.
The exception
+ * is the row type of the relational expression: the row type is needed
because during
+ * planning, new equivalent rels may be produced with changed fields
nullability
+ * (i.e. most of them comes from the rex simplify or constant reduction).
+ * This expects to be rare case, so the hashcode is computed without row type
+ * but when it conflicts, we compare with the row type involved(sans field
names).
+ *
+ * @param rel The rel to compute digest
+ * @param contents The rel properties should be considered in digest
+ * @param withType Whether to involve the row type
+ */
+ private static Object[] collect(
+ RelNode rel,
+ List> contents,
+ boolean withType) {
+List hashCodeItems = new ArrayList<>();
+// The type name.
+hashCodeItems.add(rel.getRelTypeName());
+// The traits.
+hashCodeItems.addAll(rel.getTraitSet());
+// The hints.
+if (rel instanceof Hintable) {
+ hashCodeItems.addAll(((Hintable) rel).getHints());
+}
+if
[GitHub] [calcite] danny0405 commented on pull request #2016: [CALCITE-3786] Add Digest interface to enable efficient hashCode(equa…
danny0405 commented on pull request #2016: URL: https://github.com/apache/calcite/pull/2016#issuecomment-643544569 > I have to say that I don't see how this pull request makes hash code more efficient (precomputation of hashcode?). For sure it looks to increase memory usage and also remove lots of flexibility for Calcite users extending on RelNode and RexnNode... Compare to the old code, this patch reduce the memory usage, especially for the `RexNode`. What do you mean by remove lots of flexibility for Calcite users ? Which flexibility do i remove ? This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [calcite] danny0405 commented on a change in pull request #2016: [CALCITE-3786] Add Digest interface to enable efficient hashCode(equa…
danny0405 commented on a change in pull request #2016:
URL: https://github.com/apache/calcite/pull/2016#discussion_r439692622
##
File path: core/src/main/java/org/apache/calcite/plan/Digest.java
##
@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.plan;
+
+import org.apache.calcite.plan.hep.HepRelVertex;
+import org.apache.calcite.plan.volcano.RelSubset;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.hint.Hintable;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.util.Pair;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * A short description of relational expression's type, inputs, and
+ * other properties. The digest uniquely identifies the node; another node
+ * is equivalent if and only if it has the same value.
+ *
+ * Row type is part of the digest for the rare occasion that similar
+ * expressions have different types, e.g. variants of
+ * {@code Project(child=rel#1, a=null)} where a is a null INTEGER or a
+ * null VARCHAR(10). Row type is represented as fieldTypes only, so {@code
RelNode}
+ * that differ with field names only are treated equal.
+ * For instance, {@code Project(input=rel#1,empid=$0)} and {@code
Project(input=rel#1,deptno=$0)}
+ * are equal.
+ *
+ * Computed by {@code org.apache.calcite.rel.AbstractRelNode#computeDigest},
+ * assigned by {@link org.apache.calcite.rel.AbstractRelNode#onRegister},
+ * returned by {@link org.apache.calcite.rel.AbstractRelNode#getDigest()}.
+ */
+public class Digest implements Comparable {
+
+ //~ Instance fields
+
+ final int hashCode;
+ final List> items;
+ private final RelNode rel;
+
+ // Used for debugging, computed lazily.
+ private String digest = null;
+
+ //~ Constructors ---
+
+ /**
+ * Creates a digest with given rel and properties.
+ *
+ * @param rel The rel
+ * @param items The properties, e.g. the inputs, the type, the traits and so
on
+ */
+ private Digest(RelNode rel, List> items) {
+this.rel = rel;
+this.items = normalizeContents(items);
+this.hashCode = computeIdentity(rel, this.items);
+ }
+
+ /**
+ * Creates a digest with given rel, the digest is computed as simple,
+ * see {@link #simpleRelDigest(RelNode)}.
+ */
+ private Digest(RelNode rel) {
+this(rel, simpleRelDigest(rel));
+ }
+
+ /** Creates a digest with given rel and string format digest. */
+ private Digest(RelNode rel, String digest) {
+this.rel = rel;
+this.items = Collections.emptyList();
+this.digest = digest;
+this.hashCode = this.digest.hashCode();
+ }
+
+ /** Returns the identity of this digest which is used to speedup hashCode
and equals. */
+ private static int computeIdentity(RelNode rel, List>
contents) {
+return Objects.hash(collect(rel, contents, false));
+ }
+
+ /**
+ * Collects the items used for {@link #hashCode} and {@link #equals}.
+ *
+ * Generally, the items used for hashCode and equals should be the same.
The exception
+ * is the row type of the relational expression: the row type is needed
because during
+ * planning, new equivalent rels may be produced with changed fields
nullability
+ * (i.e. most of them comes from the rex simplify or constant reduction).
+ * This expects to be rare case, so the hashcode is computed without row type
+ * but when it conflicts, we compare with the row type involved(sans field
names).
+ *
+ * @param rel The rel to compute digest
+ * @param contents The rel properties should be considered in digest
+ * @param withType Whether to involve the row type
+ */
+ private static Object[] collect(
+ RelNode rel,
+ List> contents,
+ boolean withType) {
+List hashCodeItems = new ArrayList<>();
+// The type name.
+hashCodeItems.add(rel.getRelTypeName());
+// The traits.
+hashCodeItems.addAll(rel.getTraitSet());
+// The hints.
+if (rel instanceof Hintable) {
+ hashCodeItems.addAll(((Hintable) rel).getHints());
+}
+if
[GitHub] [calcite] danny0405 commented on a change in pull request #2016: [CALCITE-3786] Add Digest interface to enable efficient hashCode(equa…
danny0405 commented on a change in pull request #2016:
URL: https://github.com/apache/calcite/pull/2016#discussion_r439692488
##
File path: core/src/main/java/org/apache/calcite/plan/Digest.java
##
@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.plan;
+
+import org.apache.calcite.plan.hep.HepRelVertex;
+import org.apache.calcite.plan.volcano.RelSubset;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.hint.Hintable;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.util.Pair;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * A short description of relational expression's type, inputs, and
+ * other properties. The digest uniquely identifies the node; another node
+ * is equivalent if and only if it has the same value.
+ *
+ * Row type is part of the digest for the rare occasion that similar
+ * expressions have different types, e.g. variants of
+ * {@code Project(child=rel#1, a=null)} where a is a null INTEGER or a
+ * null VARCHAR(10). Row type is represented as fieldTypes only, so {@code
RelNode}
+ * that differ with field names only are treated equal.
+ * For instance, {@code Project(input=rel#1,empid=$0)} and {@code
Project(input=rel#1,deptno=$0)}
+ * are equal.
+ *
+ * Computed by {@code org.apache.calcite.rel.AbstractRelNode#computeDigest},
+ * assigned by {@link org.apache.calcite.rel.AbstractRelNode#onRegister},
+ * returned by {@link org.apache.calcite.rel.AbstractRelNode#getDigest()}.
+ */
+public class Digest implements Comparable {
+
+ //~ Instance fields
+
+ final int hashCode;
+ final List> items;
+ private final RelNode rel;
+
+ // Used for debugging, computed lazily.
+ private String digest = null;
+
+ //~ Constructors ---
+
+ /**
+ * Creates a digest with given rel and properties.
+ *
+ * @param rel The rel
+ * @param items The properties, e.g. the inputs, the type, the traits and so
on
+ */
+ private Digest(RelNode rel, List> items) {
+this.rel = rel;
+this.items = normalizeContents(items);
+this.hashCode = computeIdentity(rel, this.items);
+ }
+
+ /**
+ * Creates a digest with given rel, the digest is computed as simple,
+ * see {@link #simpleRelDigest(RelNode)}.
+ */
+ private Digest(RelNode rel) {
+this(rel, simpleRelDigest(rel));
+ }
+
+ /** Creates a digest with given rel and string format digest. */
+ private Digest(RelNode rel, String digest) {
+this.rel = rel;
+this.items = Collections.emptyList();
+this.digest = digest;
+this.hashCode = this.digest.hashCode();
+ }
+
+ /** Returns the identity of this digest which is used to speedup hashCode
and equals. */
+ private static int computeIdentity(RelNode rel, List>
contents) {
+return Objects.hash(collect(rel, contents, false));
+ }
+
+ /**
+ * Collects the items used for {@link #hashCode} and {@link #equals}.
+ *
+ * Generally, the items used for hashCode and equals should be the same.
The exception
+ * is the row type of the relational expression: the row type is needed
because during
+ * planning, new equivalent rels may be produced with changed fields
nullability
+ * (i.e. most of them comes from the rex simplify or constant reduction).
+ * This expects to be rare case, so the hashcode is computed without row type
+ * but when it conflicts, we compare with the row type involved(sans field
names).
+ *
+ * @param rel The rel to compute digest
+ * @param contents The rel properties should be considered in digest
+ * @param withType Whether to involve the row type
+ */
+ private static Object[] collect(
+ RelNode rel,
+ List> contents,
+ boolean withType) {
+List hashCodeItems = new ArrayList<>();
+// The type name.
+hashCodeItems.add(rel.getRelTypeName());
+// The traits.
+hashCodeItems.addAll(rel.getTraitSet());
+// The hints.
+if (rel instanceof Hintable) {
+ hashCodeItems.addAll(((Hintable) rel).getHints());
+}
+if
[GitHub] [calcite] danny0405 commented on a change in pull request #2016: [CALCITE-3786] Add Digest interface to enable efficient hashCode(equa…
danny0405 commented on a change in pull request #2016:
URL: https://github.com/apache/calcite/pull/2016#discussion_r439692310
##
File path: core/src/main/java/org/apache/calcite/plan/Digest.java
##
@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.plan;
+
+import org.apache.calcite.plan.hep.HepRelVertex;
+import org.apache.calcite.plan.volcano.RelSubset;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.hint.Hintable;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.util.Pair;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * A short description of relational expression's type, inputs, and
+ * other properties. The digest uniquely identifies the node; another node
+ * is equivalent if and only if it has the same value.
+ *
+ * Row type is part of the digest for the rare occasion that similar
+ * expressions have different types, e.g. variants of
+ * {@code Project(child=rel#1, a=null)} where a is a null INTEGER or a
+ * null VARCHAR(10). Row type is represented as fieldTypes only, so {@code
RelNode}
+ * that differ with field names only are treated equal.
+ * For instance, {@code Project(input=rel#1,empid=$0)} and {@code
Project(input=rel#1,deptno=$0)}
+ * are equal.
+ *
+ * Computed by {@code org.apache.calcite.rel.AbstractRelNode#computeDigest},
+ * assigned by {@link org.apache.calcite.rel.AbstractRelNode#onRegister},
+ * returned by {@link org.apache.calcite.rel.AbstractRelNode#getDigest()}.
+ */
+public class Digest implements Comparable {
+
+ //~ Instance fields
+
+ final int hashCode;
+ final List> items;
+ private final RelNode rel;
+
+ // Used for debugging, computed lazily.
+ private String digest = null;
+
+ //~ Constructors ---
+
+ /**
+ * Creates a digest with given rel and properties.
+ *
+ * @param rel The rel
+ * @param items The properties, e.g. the inputs, the type, the traits and so
on
+ */
+ private Digest(RelNode rel, List> items) {
+this.rel = rel;
+this.items = normalizeContents(items);
+this.hashCode = computeIdentity(rel, this.items);
+ }
+
+ /**
+ * Creates a digest with given rel, the digest is computed as simple,
+ * see {@link #simpleRelDigest(RelNode)}.
+ */
+ private Digest(RelNode rel) {
+this(rel, simpleRelDigest(rel));
+ }
+
+ /** Creates a digest with given rel and string format digest. */
+ private Digest(RelNode rel, String digest) {
+this.rel = rel;
+this.items = Collections.emptyList();
+this.digest = digest;
+this.hashCode = this.digest.hashCode();
+ }
+
+ /** Returns the identity of this digest which is used to speedup hashCode
and equals. */
+ private static int computeIdentity(RelNode rel, List>
contents) {
+return Objects.hash(collect(rel, contents, false));
+ }
+
+ /**
+ * Collects the items used for {@link #hashCode} and {@link #equals}.
+ *
+ * Generally, the items used for hashCode and equals should be the same.
The exception
+ * is the row type of the relational expression: the row type is needed
because during
+ * planning, new equivalent rels may be produced with changed fields
nullability
+ * (i.e. most of them comes from the rex simplify or constant reduction).
+ * This expects to be rare case, so the hashcode is computed without row type
+ * but when it conflicts, we compare with the row type involved(sans field
names).
+ *
+ * @param rel The rel to compute digest
+ * @param contents The rel properties should be considered in digest
+ * @param withType Whether to involve the row type
+ */
+ private static Object[] collect(
+ RelNode rel,
+ List> contents,
+ boolean withType) {
+List hashCodeItems = new ArrayList<>();
+// The type name.
+hashCodeItems.add(rel.getRelTypeName());
+// The traits.
+hashCodeItems.addAll(rel.getTraitSet());
+// The hints.
+if (rel instanceof Hintable) {
+ hashCodeItems.addAll(((Hintable) rel).getHints());
+}
+if
[GitHub] [calcite] julianhyde opened a new pull request #2024: [CALCITE-3923] Refactor how planner rules are parameterized
julianhyde opened a new pull request #2024: URL: https://github.com/apache/calcite/pull/2024 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [calcite] amaliujia commented on pull request #2006: [CALCITE-4015] Pass through parent collation request on subset or sup…
amaliujia commented on pull request #2006: URL: https://github.com/apache/calcite/pull/2006#issuecomment-643500881 Friendly ping~ This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [calcite] amaliujia commented on a change in pull request #2023: [CALCITE-4057] Implement trait propagation for EnumerableBatchNestedL…
amaliujia commented on a change in pull request #2023: URL: https://github.com/apache/calcite/pull/2023#discussion_r439609801 ## File path: core/src/main/java/org/apache/calcite/adapter/enumerable/EnumTraitsUtils.java ## @@ -148,4 +149,52 @@ private static boolean isCollationOnTrivialExpr( return null; } } + + // EnumerableNestedLoopJoin and EnumerableBatchNestedLoopJoin traits passdown shall only + // pass through collation to left input. It is because for EnumerableNestedLoopJoin always + // uses left input as the outer loop, thus only left input can preserve ordering. + // Push sort both to left and right inputs does not help right outer join. It's because in + // implementation, EnumerableNestedLoopJoin produces (null, right_unmatched) all together, + // which does not preserve ordering from right side. + static Pair> passThroughTraitsForNestedLoopJoin( Review comment: Pushed a commit to make HashJoin and Correlate reuse the code. Also rename this class to `EnumerableTraitUtils` This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[calcite] branch master updated (3f049ee -> bb22d47)
This is an automated email from the ASF dual-hosted git repository. vladimirsitnikov pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/calcite.git. omit 3f049ee Update Gradle: 6.3 -> 6.5 This update removed existing revisions from the reference, leaving the reference pointing at a previous point in the repository history. * -- * -- N refs/heads/master (bb22d47) \ O -- O -- O (3f049ee) Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: babel/build.gradle.kts | 2 +- .../org/apache/calcite/buildtools/javacc/JavaCCTask.kt | 17 ++--- core/build.gradle.kts | 4 ++-- gradle/wrapper/gradle-wrapper.properties| 4 ++-- server/build.gradle.kts | 2 +- 5 files changed, 16 insertions(+), 13 deletions(-)
[calcite] branch master updated: Update Gradle: 6.3 -> 6.5
This is an automated email from the ASF dual-hosted git repository.
vladimirsitnikov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/calcite.git
The following commit(s) were added to refs/heads/master by this push:
new 3f049ee Update Gradle: 6.3 -> 6.5
3f049ee is described below
commit 3f049eed72cebb916883912affed19154a2bbcfa
Author: Vladimir Sitnikov
AuthorDate: Fri Jun 12 21:41:48 2020 +0300
Update Gradle: 6.3 -> 6.5
---
babel/build.gradle.kts | 2 +-
.../org/apache/calcite/buildtools/javacc/JavaCCTask.kt | 17 +++--
core/build.gradle.kts | 4 ++--
gradle/wrapper/gradle-wrapper.properties| 4 ++--
server/build.gradle.kts | 2 +-
5 files changed, 13 insertions(+), 16 deletions(-)
diff --git a/babel/build.gradle.kts b/babel/build.gradle.kts
index ee24594..091bcd8 100644
--- a/babel/build.gradle.kts
+++ b/babel/build.gradle.kts
@@ -46,7 +46,7 @@ val javaCCMain by
tasks.registering(org.apache.calcite.buildtools.javacc.JavaCCT
dependsOn(fmppMain)
lookAhead.set(2)
val parserFile = fmppMain.map {
-it.output.asFileTree.matching { include("**/Parser.jj") }.singleFile
+it.output.asFileTree.matching { include("**/Parser.jj")
}.singleOrNull().toPlatformType()
}
inputFile.set(parserFile)
packageName.set("org.apache.calcite.sql.parser.babel")
diff --git
a/buildSrc/subprojects/javacc/src/main/kotlin/org/apache/calcite/buildtools/javacc/JavaCCTask.kt
b/buildSrc/subprojects/javacc/src/main/kotlin/org/apache/calcite/buildtools/javacc/JavaCCTask.kt
index 4de0ea8..ced2c71 100644
---
a/buildSrc/subprojects/javacc/src/main/kotlin/org/apache/calcite/buildtools/javacc/JavaCCTask.kt
+++
b/buildSrc/subprojects/javacc/src/main/kotlin/org/apache/calcite/buildtools/javacc/JavaCCTask.kt
@@ -18,6 +18,7 @@
package org.apache.calcite.buildtools.javacc
import java.io.File
+import java.util.function.Function
import javax.inject.Inject
import org.gradle.api.DefaultTask
import org.gradle.api.artifacts.Configuration
@@ -25,7 +26,6 @@ import org.gradle.api.model.ObjectFactory
import org.gradle.api.tasks.Classpath
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.InputFile
-import org.gradle.api.tasks.Internal
import org.gradle.api.tasks.OutputDirectory
import org.gradle.api.tasks.TaskAction
import org.gradle.kotlin.dsl.property
@@ -37,17 +37,14 @@ open class JavaCCTask @Inject constructor(
val javaCCClasspath = objectFactory.property()
.convention(project.configurations.named(JavaCCPlugin.JAVACC_CLASSPATH_CONFIGURATION_NAME))
-@Internal
+@InputFile
val inputFile = objectFactory.property()
-// See https://github.com/gradle/gradle/issues/12627
-@get:InputFile
-val actualInputFile: File? get() = try {
-inputFile.get()
-} catch (e: IllegalStateException) {
-// This means Gradle queries property too early
-null
-}
+/**
+ * This enables to cast a nullable value to a non-nullable one
+ * @see [https://github.com/gradle/gradle/issues/12627]
+ */
+inline fun T?.toPlatformType(): T =
Function.identity().apply(this)
@Input
val lookAhead = objectFactory.property().convention(1)
diff --git a/core/build.gradle.kts b/core/build.gradle.kts
index 6a5c103..c4faaac 100644
--- a/core/build.gradle.kts
+++ b/core/build.gradle.kts
@@ -159,7 +159,7 @@ val fmppMain by
tasks.registering(org.apache.calcite.buildtools.fmpp.FmppTask::c
val javaCCMain by
tasks.registering(org.apache.calcite.buildtools.javacc.JavaCCTask::class) {
dependsOn(fmppMain)
val parserFile = fmppMain.map {
-it.output.asFileTree.matching { include("**/Parser.jj") }.singleFile
+it.output.asFileTree.matching { include("**/Parser.jj")
}.singleOrNull().toPlatformType()
}
inputFile.set(parserFile)
packageName.set("org.apache.calcite.sql.parser.impl")
@@ -173,7 +173,7 @@ val fmppTest by
tasks.registering(org.apache.calcite.buildtools.fmpp.FmppTask::c
val javaCCTest by
tasks.registering(org.apache.calcite.buildtools.javacc.JavaCCTask::class) {
dependsOn(fmppTest)
val parserFile = fmppTest.map {
-it.output.asFileTree.matching { include("**/Parser.jj") }.singleFile
+it.output.asFileTree.matching { include("**/Parser.jj")
}.singleOrNull().toPlatformType()
}
inputFile.set(parserFile)
packageName.set("org.apache.calcite.sql.parser.parserextensiontesting")
diff --git a/gradle/wrapper/gradle-wrapper.properties
b/gradle/wrapper/gradle-wrapper.properties
index 0db054b..2c65038 100644
--- a/gradle/wrapper/gradle-wrapper.properties
+++ b/gradle/wrapper/gradle-wrapper.properties
@@ -16,7 +16,7 @@
#
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
-distributionSha256Sum=0f316a67b971b7b571dac7215dcf2591a30994b3450e0629925ffcfe2c68cc5c
[GitHub] [calcite] hsyuan commented on a change in pull request #2023: [CALCITE-4057] Implement trait propagation for EnumerableBatchNestedL…
hsyuan commented on a change in pull request #2023: URL: https://github.com/apache/calcite/pull/2023#discussion_r439580414 ## File path: core/src/main/java/org/apache/calcite/adapter/enumerable/EnumTraitsUtils.java ## @@ -148,4 +149,52 @@ private static boolean isCollationOnTrivialExpr( return null; } } + + // EnumerableNestedLoopJoin and EnumerableBatchNestedLoopJoin traits passdown shall only + // pass through collation to left input. It is because for EnumerableNestedLoopJoin always + // uses left input as the outer loop, thus only left input can preserve ordering. + // Push sort both to left and right inputs does not help right outer join. It's because in + // implementation, EnumerableNestedLoopJoin produces (null, right_unmatched) all together, + // which does not preserve ordering from right side. + static Pair> passThroughTraitsForNestedLoopJoin( Review comment: The class name is misleading. Should be EnumerableTraitUtil, Enum is a different thing. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [calcite] amaliujia commented on a change in pull request #2023: [CALCITE-4057] Implement trait propagation for EnumerableBatchNestedL…
amaliujia commented on a change in pull request #2023: URL: https://github.com/apache/calcite/pull/2023#discussion_r439579435 ## File path: core/src/main/java/org/apache/calcite/adapter/enumerable/EnumTraitsUtils.java ## @@ -148,4 +149,52 @@ private static boolean isCollationOnTrivialExpr( return null; } } + + // EnumerableNestedLoopJoin and EnumerableBatchNestedLoopJoin traits passdown shall only + // pass through collation to left input. It is because for EnumerableNestedLoopJoin always + // uses left input as the outer loop, thus only left input can preserve ordering. + // Push sort both to left and right inputs does not help right outer join. It's because in + // implementation, EnumerableNestedLoopJoin produces (null, right_unmatched) all together, + // which does not preserve ordering from right side. + static Pair> passThroughTraitsForNestedLoopJoin( Review comment: Nice catch. Will migrate them too to this PR. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [calcite] hsyuan commented on a change in pull request #2023: [CALCITE-4057] Implement trait propagation for EnumerableBatchNestedL…
hsyuan commented on a change in pull request #2023:
URL: https://github.com/apache/calcite/pull/2023#discussion_r439578915
##
File path:
core/src/main/java/org/apache/calcite/adapter/enumerable/EnumTraitsUtils.java
##
@@ -148,4 +149,52 @@ private static boolean isCollationOnTrivialExpr(
return null;
}
}
+
+ // EnumerableNestedLoopJoin and EnumerableBatchNestedLoopJoin traits
passdown shall only
+ // pass through collation to left input. It is because for
EnumerableNestedLoopJoin always
+ // uses left input as the outer loop, thus only left input can preserve
ordering.
+ // Push sort both to left and right inputs does not help right outer join.
It's because in
+ // implementation, EnumerableNestedLoopJoin produces (null, right_unmatched)
all together,
+ // which does not preserve ordering from right side.
+ static Pair>
passThroughTraitsForNestedLoopJoin(
+ RelTraitSet required, JoinRelType joinType,
+ int leftInputFieldCount, RelTraitSet joinTraitSet) {
+RelCollation collation = required.getCollation();
+if (collation == null
+|| collation == RelCollations.EMPTY
+|| joinType == JoinRelType.FULL
+|| joinType == JoinRelType.RIGHT) {
+ return null;
+}
+
+for (RelFieldCollation fc : collation.getFieldCollations()) {
+ // If field collation belongs to right input: cannot push down collation.
+ if (fc.getFieldIndex() >= leftInputFieldCount) {
+return null;
+ }
+}
+
+RelTraitSet passthroughTraitSet = joinTraitSet.replace(collation);
+return Pair.of(passthroughTraitSet,
+ImmutableList.of(
+passthroughTraitSet,
+passthroughTraitSet.replace(RelCollations.EMPTY)));
+ }
+
+ static Pair> deriveTraitsForNestedLoopJoin(
+ RelTraitSet childTraits, int childId,
+ RelTraitSet joinTraitSet, RelTraitSet rightTraitSet) {
+// should only derive traits (limited to collation for now) from left join
input.
+assert childId == 0;
+
+RelCollation collation = childTraits.getCollation();
+if (collation == null || collation == RelCollations.EMPTY) {
Review comment:
Add check
```
|| joinType == JoinRelType.FULL
|| joinType == JoinRelType.RIGHT
```
##
File path:
core/src/main/java/org/apache/calcite/adapter/enumerable/EnumTraitsUtils.java
##
@@ -148,4 +149,52 @@ private static boolean isCollationOnTrivialExpr(
return null;
}
}
+
+ // EnumerableNestedLoopJoin and EnumerableBatchNestedLoopJoin traits
passdown shall only
+ // pass through collation to left input. It is because for
EnumerableNestedLoopJoin always
+ // uses left input as the outer loop, thus only left input can preserve
ordering.
+ // Push sort both to left and right inputs does not help right outer join.
It's because in
+ // implementation, EnumerableNestedLoopJoin produces (null, right_unmatched)
all together,
+ // which does not preserve ordering from right side.
+ static Pair>
passThroughTraitsForNestedLoopJoin(
Review comment:
HashJoin and Correlate can reuse it too.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [calcite] amaliujia commented on a change in pull request #2023: [CALCITE-4057] Implement trait propagation for EnumerableBatchNestedL…
amaliujia commented on a change in pull request #2023:
URL: https://github.com/apache/calcite/pull/2023#discussion_r439577517
##
File path: core/src/test/java/org/apache/calcite/test/TopDownOptTest.java
##
@@ -693,6 +678,37 @@
.removeRule(EnumerableRules.ENUMERABLE_JOIN_RULE)
.check();
}
+
+ // push sort to left input
+ @Test void testBatchNestedLoopJoinLeftOuterJoinPushDownSort() {
Review comment:
Not adding too many tests for EnumerableBatchNestedLoopJoin because it
reuses EnumerableNestedLoopJoin's code and those code was tested.
Add two test only to verify that trait propagation in
EnumerableBatchNestedLoopJoin is working.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [calcite] amaliujia commented on a change in pull request #2023: [CALCITE-4057] Implement trait propagation for EnumerableBatchNestedL…
amaliujia commented on a change in pull request #2023: URL: https://github.com/apache/calcite/pull/2023#discussion_r439577088 ## File path: core/src/test/java/org/apache/calcite/test/TopDownOptTest.java ## @@ -483,7 +476,6 @@ Query.create(sql) .removeRule(EnumerableRules.ENUMERABLE_MERGE_JOIN_RULE) -.removeRule(EnumerableRules.ENUMERABLE_BATCH_NESTED_LOOP_JOIN_RULE) Review comment: My bad. This rule is not added by default. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [calcite] amaliujia commented on pull request #2023: [CALCITE-4057] Implement trait propagation for EnumerableBatchNestedL…
amaliujia commented on pull request #2023: URL: https://github.com/apache/calcite/pull/2023#issuecomment-643421126 R: @chunweilei @hsyuan This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [calcite] amaliujia opened a new pull request #2023: [CALCITE-4057] Implement trait propagation for EnumerableBatchNestedL…
amaliujia opened a new pull request #2023: URL: https://github.com/apache/calcite/pull/2023 …oopJoin (Rui Wang). This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [calcite] laurentgo commented on a change in pull request #2016: [CALCITE-3786] Add Digest interface to enable efficient hashCode(equa…
laurentgo commented on a change in pull request #2016:
URL: https://github.com/apache/calcite/pull/2016#discussion_r439507991
##
File path: core/src/main/java/org/apache/calcite/plan/Digest.java
##
@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.plan;
+
+import org.apache.calcite.plan.hep.HepRelVertex;
+import org.apache.calcite.plan.volcano.RelSubset;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.hint.Hintable;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.util.Pair;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * A short description of relational expression's type, inputs, and
+ * other properties. The digest uniquely identifies the node; another node
+ * is equivalent if and only if it has the same value.
+ *
+ * Row type is part of the digest for the rare occasion that similar
+ * expressions have different types, e.g. variants of
+ * {@code Project(child=rel#1, a=null)} where a is a null INTEGER or a
+ * null VARCHAR(10). Row type is represented as fieldTypes only, so {@code
RelNode}
+ * that differ with field names only are treated equal.
+ * For instance, {@code Project(input=rel#1,empid=$0)} and {@code
Project(input=rel#1,deptno=$0)}
+ * are equal.
+ *
+ * Computed by {@code org.apache.calcite.rel.AbstractRelNode#computeDigest},
+ * assigned by {@link org.apache.calcite.rel.AbstractRelNode#onRegister},
+ * returned by {@link org.apache.calcite.rel.AbstractRelNode#getDigest()}.
+ */
+public class Digest implements Comparable {
+
+ //~ Instance fields
+
+ final int hashCode;
+ final List> items;
+ private final RelNode rel;
+
+ // Used for debugging, computed lazily.
+ private String digest = null;
+
+ //~ Constructors ---
+
+ /**
+ * Creates a digest with given rel and properties.
+ *
+ * @param rel The rel
+ * @param items The properties, e.g. the inputs, the type, the traits and so
on
+ */
+ private Digest(RelNode rel, List> items) {
+this.rel = rel;
+this.items = normalizeContents(items);
+this.hashCode = computeIdentity(rel, this.items);
+ }
+
+ /**
+ * Creates a digest with given rel, the digest is computed as simple,
+ * see {@link #simpleRelDigest(RelNode)}.
+ */
+ private Digest(RelNode rel) {
+this(rel, simpleRelDigest(rel));
+ }
+
+ /** Creates a digest with given rel and string format digest. */
+ private Digest(RelNode rel, String digest) {
+this.rel = rel;
+this.items = Collections.emptyList();
+this.digest = digest;
+this.hashCode = this.digest.hashCode();
+ }
+
+ /** Returns the identity of this digest which is used to speedup hashCode
and equals. */
+ private static int computeIdentity(RelNode rel, List>
contents) {
+return Objects.hash(collect(rel, contents, false));
+ }
+
+ /**
+ * Collects the items used for {@link #hashCode} and {@link #equals}.
+ *
+ * Generally, the items used for hashCode and equals should be the same.
The exception
+ * is the row type of the relational expression: the row type is needed
because during
+ * planning, new equivalent rels may be produced with changed fields
nullability
+ * (i.e. most of them comes from the rex simplify or constant reduction).
+ * This expects to be rare case, so the hashcode is computed without row type
+ * but when it conflicts, we compare with the row type involved(sans field
names).
+ *
+ * @param rel The rel to compute digest
+ * @param contents The rel properties should be considered in digest
+ * @param withType Whether to involve the row type
+ */
+ private static Object[] collect(
+ RelNode rel,
+ List> contents,
+ boolean withType) {
+List hashCodeItems = new ArrayList<>();
+// The type name.
+hashCodeItems.add(rel.getRelTypeName());
+// The traits.
+hashCodeItems.addAll(rel.getTraitSet());
+// The hints.
+if (rel instanceof Hintable) {
+ hashCodeItems.addAll(((Hintable) rel).getHints());
+}
+if
[calcite] branch master updated (f099919 -> bb22d47)
This is an automated email from the ASF dual-hosted git repository. vladimirsitnikov pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/calcite.git. omit f099919 [CALCITE-4061] Build should fail if Calcite code uses deprecated APIs new bb22d47 [CALCITE-4061] Build should fail if Calcite code uses deprecated APIs This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (f099919) \ N -- N -- N refs/heads/master (bb22d47) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java| 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-)
[calcite] 01/01: [CALCITE-4061] Build should fail if Calcite code uses deprecated APIs
This is an automated email from the ASF dual-hosted git repository.
vladimirsitnikov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/calcite.git
commit bb22d4773cd2c24bb69acaaf548fb8d17ec99182
Author: Vladimir Sitnikov
AuthorDate: Fri Jun 12 15:24:43 2020 +0300
[CALCITE-4061] Build should fail if Calcite code uses deprecated APIs
---
build.gradle.kts | 1 +
.../org/apache/calcite/adapter/cassandra/CassandraSchema.java | 8 +++-
core/src/main/java/org/apache/calcite/rel/core/Window.java| 2 +-
core/src/main/java/org/apache/calcite/sql/SqlOverOperator.java| 2 +-
core/src/main/java/org/apache/calcite/sql/SqlWindow.java | 2 +-
.../java/org/apache/calcite/sql/validate/SqlValidatorImpl.java| 3 ++-
core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java | 5 +++--
.../main/java/org/apache/calcite/adapter/druid/DruidRules.java| 3 +--
.../main/java/org/apache/calcite/adapter/spark/HttpServer.java| 1 -
9 files changed, 17 insertions(+), 10 deletions(-)
diff --git a/build.gradle.kts b/build.gradle.kts
index af8e0ab..7512783 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -523,6 +523,7 @@ allprojects {
configureEach {
options.encoding = "UTF-8"
+options.compilerArgs.addAll(listOf("-Xlint:deprecation",
"-Werror"))
}
configureEach {
useJUnitPlatform {
diff --git
a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraSchema.java
b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraSchema.java
index b0a2458..17b5c70 100644
---
a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraSchema.java
+++
b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraSchema.java
@@ -149,7 +149,13 @@ public class CassandraSchema extends AbstractSchema {
this.parentSchema = parentSchema;
this.name = name;
-this.hook = Hook.TRIMMED.add(node -> {
+this.hook = prepareHook();
+ }
+
+ @SuppressWarnings("deprecation")
+ private Hook.Closeable prepareHook() {
+// It adds a global hook, so it should probably be replaced with a
thread-local hook
+return Hook.TRIMMED.add(node -> {
CassandraSchema.this.addMaterializedViews();
});
}
diff --git a/core/src/main/java/org/apache/calcite/rel/core/Window.java
b/core/src/main/java/org/apache/calcite/rel/core/Window.java
index 13d8830..71176e5 100644
--- a/core/src/main/java/org/apache/calcite/rel/core/Window.java
+++ b/core/src/main/java/org/apache/calcite/rel/core/Window.java
@@ -308,7 +308,7 @@ public abstract class Window extends SingleRel {
* @return true when the window is non-empty
* @see org.apache.calcite.sql.SqlWindow#isAlwaysNonEmpty()
* @see org.apache.calcite.sql.SqlOperatorBinding#getGroupCount()
- * @see
org.apache.calcite.sql.validate.SqlValidatorImpl#resolveWindow(org.apache.calcite.sql.SqlNode,
org.apache.calcite.sql.validate.SqlValidatorScope, boolean)
+ * @see
org.apache.calcite.sql.validate.SqlValidatorImpl#resolveWindow(org.apache.calcite.sql.SqlNode,
org.apache.calcite.sql.validate.SqlValidatorScope)
*/
public boolean isAlwaysNonEmpty() {
int lowerKey = lowerBound.getOrderKey();
diff --git a/core/src/main/java/org/apache/calcite/sql/SqlOverOperator.java
b/core/src/main/java/org/apache/calcite/sql/SqlOverOperator.java
index 4e8d035..5f6cb3f 100644
--- a/core/src/main/java/org/apache/calcite/sql/SqlOverOperator.java
+++ b/core/src/main/java/org/apache/calcite/sql/SqlOverOperator.java
@@ -98,7 +98,7 @@ public class SqlOverOperator extends SqlBinaryOperator {
}
SqlNode window = call.operand(1);
-SqlWindow w = validator.resolveWindow(window, scope, false);
+SqlWindow w = validator.resolveWindow(window, scope);
final int groupCount = w.isAlwaysNonEmpty() ? 1 : 0;
final SqlCall aggCall = (SqlCall) agg;
diff --git a/core/src/main/java/org/apache/calcite/sql/SqlWindow.java
b/core/src/main/java/org/apache/calcite/sql/SqlWindow.java
index 4f816d0..68c429a 100644
--- a/core/src/main/java/org/apache/calcite/sql/SqlWindow.java
+++ b/core/src/main/java/org/apache/calcite/sql/SqlWindow.java
@@ -535,7 +535,7 @@ public class SqlWindow extends SqlCall {
SqlLiteral allowPartial = this.allowPartial;
if (refName != null) {
- SqlWindow win = validator.resolveWindow(this, operandScope, false);
+ SqlWindow win = validator.resolveWindow(this, operandScope);
partitionList = win.partitionList;
orderList = win.orderList;
isRows = win.isRows;
diff --git
a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
index c0843ca..5488898 100644
--- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
+++
[calcite] branch master updated: [CALCITE-4061] Build should fail if Calcite code uses deprecated APIs
This is an automated email from the ASF dual-hosted git repository.
vladimirsitnikov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/calcite.git
The following commit(s) were added to refs/heads/master by this push:
new f099919 [CALCITE-4061] Build should fail if Calcite code uses
deprecated APIs
f099919 is described below
commit f09991959dea41b8805d8cdd18572803d42e3738
Author: Vladimir Sitnikov
AuthorDate: Fri Jun 12 15:24:43 2020 +0300
[CALCITE-4061] Build should fail if Calcite code uses deprecated APIs
---
build.gradle.kts | 1 +
.../org/apache/calcite/adapter/cassandra/CassandraSchema.java | 8 +++-
core/src/main/java/org/apache/calcite/rel/core/Window.java| 2 +-
core/src/main/java/org/apache/calcite/sql/SqlOverOperator.java| 2 +-
core/src/main/java/org/apache/calcite/sql/SqlWindow.java | 2 +-
.../java/org/apache/calcite/sql/validate/SqlValidatorImpl.java| 2 ++
core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java | 5 +++--
.../main/java/org/apache/calcite/adapter/druid/DruidRules.java| 3 +--
.../main/java/org/apache/calcite/adapter/spark/HttpServer.java| 1 -
9 files changed, 17 insertions(+), 9 deletions(-)
diff --git a/build.gradle.kts b/build.gradle.kts
index af8e0ab..7512783 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -523,6 +523,7 @@ allprojects {
configureEach {
options.encoding = "UTF-8"
+options.compilerArgs.addAll(listOf("-Xlint:deprecation",
"-Werror"))
}
configureEach {
useJUnitPlatform {
diff --git
a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraSchema.java
b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraSchema.java
index b0a2458..17b5c70 100644
---
a/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraSchema.java
+++
b/cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraSchema.java
@@ -149,7 +149,13 @@ public class CassandraSchema extends AbstractSchema {
this.parentSchema = parentSchema;
this.name = name;
-this.hook = Hook.TRIMMED.add(node -> {
+this.hook = prepareHook();
+ }
+
+ @SuppressWarnings("deprecation")
+ private Hook.Closeable prepareHook() {
+// It adds a global hook, so it should probably be replaced with a
thread-local hook
+return Hook.TRIMMED.add(node -> {
CassandraSchema.this.addMaterializedViews();
});
}
diff --git a/core/src/main/java/org/apache/calcite/rel/core/Window.java
b/core/src/main/java/org/apache/calcite/rel/core/Window.java
index 13d8830..71176e5 100644
--- a/core/src/main/java/org/apache/calcite/rel/core/Window.java
+++ b/core/src/main/java/org/apache/calcite/rel/core/Window.java
@@ -308,7 +308,7 @@ public abstract class Window extends SingleRel {
* @return true when the window is non-empty
* @see org.apache.calcite.sql.SqlWindow#isAlwaysNonEmpty()
* @see org.apache.calcite.sql.SqlOperatorBinding#getGroupCount()
- * @see
org.apache.calcite.sql.validate.SqlValidatorImpl#resolveWindow(org.apache.calcite.sql.SqlNode,
org.apache.calcite.sql.validate.SqlValidatorScope, boolean)
+ * @see
org.apache.calcite.sql.validate.SqlValidatorImpl#resolveWindow(org.apache.calcite.sql.SqlNode,
org.apache.calcite.sql.validate.SqlValidatorScope)
*/
public boolean isAlwaysNonEmpty() {
int lowerKey = lowerBound.getOrderKey();
diff --git a/core/src/main/java/org/apache/calcite/sql/SqlOverOperator.java
b/core/src/main/java/org/apache/calcite/sql/SqlOverOperator.java
index 4e8d035..5f6cb3f 100644
--- a/core/src/main/java/org/apache/calcite/sql/SqlOverOperator.java
+++ b/core/src/main/java/org/apache/calcite/sql/SqlOverOperator.java
@@ -98,7 +98,7 @@ public class SqlOverOperator extends SqlBinaryOperator {
}
SqlNode window = call.operand(1);
-SqlWindow w = validator.resolveWindow(window, scope, false);
+SqlWindow w = validator.resolveWindow(window, scope);
final int groupCount = w.isAlwaysNonEmpty() ? 1 : 0;
final SqlCall aggCall = (SqlCall) agg;
diff --git a/core/src/main/java/org/apache/calcite/sql/SqlWindow.java
b/core/src/main/java/org/apache/calcite/sql/SqlWindow.java
index 4f816d0..68c429a 100644
--- a/core/src/main/java/org/apache/calcite/sql/SqlWindow.java
+++ b/core/src/main/java/org/apache/calcite/sql/SqlWindow.java
@@ -535,7 +535,7 @@ public class SqlWindow extends SqlCall {
SqlLiteral allowPartial = this.allowPartial;
if (refName != null) {
- SqlWindow win = validator.resolveWindow(this, operandScope, false);
+ SqlWindow win = validator.resolveWindow(this, operandScope);
partitionList = win.partitionList;
orderList = win.orderList;
isRows = win.isRows;
diff --git
a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java
[GitHub] [calcite] xy2953396112 opened a new pull request #2022: [CALCITE-4062] Support deserialize UDF array type from json string
xy2953396112 opened a new pull request #2022: URL: https://github.com/apache/calcite/pull/2022 https://issues.apache.org/jira/projects/CALCITE/issues/CALCITE-4062 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [calcite] jinxing64 commented on pull request #2013: [CALCITE-3935] Materialization-Failed, when querying with LeftJoinWithFilter
jinxing64 commented on pull request #2013: URL: https://github.com/apache/calcite/pull/2013#issuecomment-643206255 Please refine the commit message. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [calcite] jinxing64 commented on a change in pull request #2013: [CALCITE-3935] Materialization-Failed, when querying with LeftJoinWithFilter
jinxing64 commented on a change in pull request #2013:
URL: https://github.com/apache/calcite/pull/2013#discussion_r439321177
##
File path: core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java
##
@@ -1202,8 +1202,9 @@ private JoinOnLeftCalcToJoinUnifyRule() {
if (joinRelType == null) {
return null;
}
- if (joinRelType != JoinRelType.INNER
- && !(joinRelType.isOuterJoin() && qInput0Cond.isAlwaysTrue())) {
+ // If JoinType is inner-join or out-join,
+ // Calc which is bottom join can be pulled up.
+ if (joinRelType != JoinRelType.INNER && !joinRelType.isOuterJoin()) {
Review comment:
What if it's a right Join but with Calc as the left child ?
If above concern is valid, please add a corresponding test.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [calcite] danny0405 commented on pull request #2020: [CALCITE-4060] TypeCoercionImpl#inOperationCoercion consider "NOT_IN".
danny0405 commented on pull request #2020: URL: https://github.com/apache/calcite/pull/2020#issuecomment-643150807 Thanks for the contribution, can we change the title to "Supports implicit type coercion for NOT IN" which is more readable. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[calcite] branch master updated: [CALCITE-4016] Support trait propagation for EnumerableCalc
This is an automated email from the ASF dual-hosted git repository.
chunwei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/calcite.git
The following commit(s) were added to refs/heads/master by this push:
new 3c317b6 [CALCITE-4016] Support trait propagation for EnumerableCalc
3c317b6 is described below
commit 3c317b608344ab2ebd6d65b67f647bddf87376b3
Author: Chunwei Lei
AuthorDate: Tue Jun 9 11:32:28 2020 +0800
[CALCITE-4016] Support trait propagation for EnumerableCalc
---
.../adapter/enumerable/EnumTraitsUtils.java| 151
.../calcite/adapter/enumerable/EnumerableCalc.java | 20 +++
.../adapter/enumerable/EnumerableProject.java | 108 +---
.../org/apache/calcite/test/TopDownOptTest.java| 108
.../org/apache/calcite/test/TopDownOptTest.xml | 193 +
5 files changed, 476 insertions(+), 104 deletions(-)
diff --git
a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumTraitsUtils.java
b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumTraitsUtils.java
new file mode 100644
index 000..2be158a
--- /dev/null
+++
b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumTraitsUtils.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.adapter.enumerable;
+
+import org.apache.calcite.linq4j.Ord;
+import org.apache.calcite.plan.RelOptUtil;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.RelCollation;
+import org.apache.calcite.rel.RelCollations;
+import org.apache.calcite.rel.RelFieldCollation;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexCallBinding;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexUtil;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.validate.SqlMonotonicity;
+import org.apache.calcite.util.Pair;
+import org.apache.calcite.util.mapping.MappingType;
+import org.apache.calcite.util.mapping.Mappings;
+
+import com.google.common.collect.ImmutableList;
+
+import org.apiguardian.api.API;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Utilities for traits propagation.
+ */
+@API(since = "1.24", status = API.Status.INTERNAL)
+class EnumTraitsUtils {
+
+ private EnumTraitsUtils() {}
+
+ /**
+ * Determine whether there is mapping between project input and output
fields.
+ * Bail out if sort relies on non-trivial expressions.
+ */
+ private static boolean isCollationOnTrivialExpr(
+ List projects, RelDataTypeFactory typeFactory,
+ Mappings.TargetMapping map, RelFieldCollation fc, boolean passDown) {
+final int index = fc.getFieldIndex();
+int target = map.getTargetOpt(index);
+if (target < 0) {
+ return false;
+}
+
+final RexNode node = passDown ? projects.get(index) : projects.get(target);
+if (node.isA(SqlKind.CAST)) {
+ // Check whether it is a monotonic preserving cast
+ final RexCall cast = (RexCall) node;
+ RelFieldCollation newFieldCollation =
Objects.requireNonNull(RexUtil.apply(map, fc));
+ final RexCallBinding binding =
+ RexCallBinding.create(typeFactory, cast,
+ ImmutableList.of(RelCollations.of(newFieldCollation)));
+ if (cast.getOperator().getMonotonicity(binding)
+ == SqlMonotonicity.NOT_MONOTONIC) {
+return false;
+ }
+}
+
+return true;
+ }
+
+ static Pair> passThroughTraitsForProject(
+ RelTraitSet required,
+ List exps,
+ RelDataType inputRowType,
+ RelDataTypeFactory typeFactory,
+ RelTraitSet currentTraits) {
+final RelCollation collation = required.getCollation();
+if (collation == null || collation == RelCollations.EMPTY) {
+ return null;
+}
+
+final Mappings.TargetMapping map =
+RelOptUtil.permutationIgnoreCast(
+exps, inputRowType);
+
+if (collation.getFieldCollations().stream().anyMatch(
+rc -> !isCollationOnTrivialExpr(exps,
[GitHub] [calcite] chunweilei merged pull request #2011: [CALCITE-4016] Support trait propagation for EnumerableCalc
chunweilei merged pull request #2011: URL: https://github.com/apache/calcite/pull/2011 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [calcite] chunweilei commented on pull request #2011: [CALCITE-4016] Support trait propagation for EnumerableCalc
chunweilei commented on pull request #2011: URL: https://github.com/apache/calcite/pull/2011#issuecomment-643146964 Thank you for your review, @amaliujia @hsyuan . This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [calcite] danny0405 commented on a change in pull request #2016: [CALCITE-3786] Add Digest interface to enable efficient hashCode(equa…
danny0405 commented on a change in pull request #2016:
URL: https://github.com/apache/calcite/pull/2016#discussion_r439280300
##
File path: core/src/main/java/org/apache/calcite/plan/Digest.java
##
@@ -0,0 +1,245 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.plan;
+
+import org.apache.calcite.plan.hep.HepRelVertex;
+import org.apache.calcite.plan.volcano.RelSubset;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.hint.Hintable;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.util.Pair;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * A short description of relational expression's type, inputs, and
+ * other properties. The digest uniquely identifies the node; another node
+ * is equivalent if and only if it has the same value.
+ *
+ * Row type is part of the digest for the rare occasion that similar
+ * expressions have different types, e.g. variants of
+ * {@code Project(child=rel#1, a=null)} where a is a null INTEGER or a
+ * null VARCHAR(10). Row type is represented as fieldTypes only, so {@code
RelNode}
+ * that differ with field names only are treated equal.
+ * For instance, {@code Project(input=rel#1,empid=$0)} and {@code
Project(input=rel#1,deptno=$0)}
+ * are equal.
+ *
+ * Computed by {@code org.apache.calcite.rel.AbstractRelNode#computeDigest},
+ * assigned by {@link org.apache.calcite.rel.AbstractRelNode#onRegister},
+ * returned by {@link org.apache.calcite.rel.AbstractRelNode#getDigest()}.
+ */
+public class Digest implements Comparable {
+
+ //~ Instance fields
+
+ final int hashCode;
+ final List> items;
+ private final RelNode rel;
+
+ // Used for debugging, computed lazily.
+ private String digest = null;
+
+ //~ Constructors ---
+
+ /**
+ * Creates a digest with given rel and properties.
+ *
+ * @param rel The rel
+ * @param items The properties, e.g. the inputs, the type, the traits and so
on
+ */
+ private Digest(RelNode rel, List> items) {
+this.rel = rel;
+this.items = normalizeContents(items);
+this.hashCode = computeIdentity(rel, this.items);
+ }
+
+ /**
+ * Creates a digest with given rel, the digest is computed as simple,
+ * see {@link #simpleRelDigest(RelNode)}.
+ */
+ private Digest(RelNode rel) {
+this(rel, simpleRelDigest(rel));
+ }
+
+ /** Creates a digest with given rel and string format digest. */
+ private Digest(RelNode rel, String digest) {
+this.rel = rel;
+this.items = Collections.emptyList();
+this.digest = digest;
+this.hashCode = this.digest.hashCode();
+ }
+
+ /** Returns the identity of this digest which is used to speedup hashCode
and equals. */
+ private static int computeIdentity(RelNode rel, List>
contents) {
+return Objects.hash(collect(rel, contents));
+ }
+
+ private static Object[] collect(RelNode rel, List>
contents) {
Review comment:
Finally i decide to keep it as is, because the conflict is very few, i
also remove the row type when computing hashcode, because row type diff for
equiv node is also rare case(usually because of nullability changes).
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [calcite] rubenada commented on a change in pull request #1987: [CALCITE-4020] Support Calc operator in RelFieldTrimmer
rubenada commented on a change in pull request #1987:
URL: https://github.com/apache/calcite/pull/1987#discussion_r439258169
##
File path: core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java
##
@@ -350,6 +354,100 @@ public TrimResult trimFields(
Mappings.createIdentity(rel.getRowType().getFieldCount()));
}
+ /**
+ * Variant of {@link #trimFields(RelNode, ImmutableBitSet, Set)} for
+ * {@link org.apache.calcite.rel.logical.LogicalCalc}.
+ */
+ public TrimResult trimFields(
+ Calc calc,
+ ImmutableBitSet fieldsUsed,
+ Set extraFields) {
+final RexProgram rexProgram = calc.getProgram();
+final List projs = Lists.transform(rexProgram.getProjectList(),
+rexProgram::expandLocalRef);
+
+final RelDataType rowType = calc.getRowType();
+final int fieldCount = rowType.getFieldCount();
+final RelNode input = calc.getInput();
+
+final Set inputExtraFields =
+new LinkedHashSet<>(extraFields);
+RelOptUtil.InputFinder inputFinder =
+new RelOptUtil.InputFinder(inputExtraFields);
+for (Ord ord : Ord.zip(projs)) {
+ if (fieldsUsed.get(ord.i)) {
+ord.e.accept(inputFinder);
+ }
+}
+ImmutableBitSet inputFieldsUsed = inputFinder.inputBitSet.build();
+
+// Create input with trimmed columns.
+TrimResult trimResult =
+trimChild(calc, input, inputFieldsUsed, inputExtraFields);
+RelNode newInput = trimResult.left;
+final Mapping inputMapping = trimResult.right;
+
+// If the input is unchanged, and we need to project all columns,
+// there's nothing we can do.
+if (newInput == input
+&& fieldsUsed.cardinality() == fieldCount) {
+ return result(calc, Mappings.createIdentity(fieldCount));
+}
+
+// Some parts of the system can't handle rows with zero fields, so
+// pretend that one field is used.
+if (fieldsUsed.cardinality() == 0) {
+ return dummyProject(fieldCount, newInput);
Review comment:
I am not sure if, in this situation, the hints from the original Calc
must / can be transferred into the new "dummy Project". What do you think
@danny0405 ?
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [calcite] rubenada commented on a change in pull request #1987: [CALCITE-4020] Support Calc operator in RelFieldTrimmer
rubenada commented on a change in pull request #1987:
URL: https://github.com/apache/calcite/pull/1987#discussion_r439257255
##
File path: core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java
##
@@ -350,6 +354,100 @@ public TrimResult trimFields(
Mappings.createIdentity(rel.getRowType().getFieldCount()));
}
+ /**
+ * Variant of {@link #trimFields(RelNode, ImmutableBitSet, Set)} for
+ * {@link org.apache.calcite.rel.logical.LogicalCalc}.
+ */
+ public TrimResult trimFields(
+ Calc calc,
+ ImmutableBitSet fieldsUsed,
+ Set extraFields) {
+final RexProgram rexProgram = calc.getProgram();
+final List projs = Lists.transform(rexProgram.getProjectList(),
+rexProgram::expandLocalRef);
+
+final RelDataType rowType = calc.getRowType();
+final int fieldCount = rowType.getFieldCount();
+final RelNode input = calc.getInput();
+
+final Set inputExtraFields =
+new LinkedHashSet<>(extraFields);
+RelOptUtil.InputFinder inputFinder =
+new RelOptUtil.InputFinder(inputExtraFields);
+for (Ord ord : Ord.zip(projs)) {
+ if (fieldsUsed.get(ord.i)) {
+ord.e.accept(inputFinder);
+ }
+}
+ImmutableBitSet inputFieldsUsed = inputFinder.inputBitSet.build();
+
+// Create input with trimmed columns.
+TrimResult trimResult =
+trimChild(calc, input, inputFieldsUsed, inputExtraFields);
+RelNode newInput = trimResult.left;
+final Mapping inputMapping = trimResult.right;
+
+// If the input is unchanged, and we need to project all columns,
+// there's nothing we can do.
+if (newInput == input
+&& fieldsUsed.cardinality() == fieldCount) {
+ return result(calc, Mappings.createIdentity(fieldCount));
+}
+
+// Some parts of the system can't handle rows with zero fields, so
+// pretend that one field is used.
+if (fieldsUsed.cardinality() == 0) {
+ return dummyProject(fieldCount, newInput);
+}
+
+// Build new project expressions, and populate the mapping.
+final List newProjects = new ArrayList<>();
+final RexVisitor shuttle =
+new RexPermuteInputsShuttle(
+inputMapping, newInput);
+final Mapping mapping =
+Mappings.create(
+MappingType.INVERSE_SURJECTION,
+fieldCount,
+fieldsUsed.cardinality());
+for (Ord ord : Ord.zip(projs)) {
+ if (fieldsUsed.get(ord.i)) {
+mapping.set(ord.i, newProjects.size());
+RexNode newProjectExpr = ord.e.accept(shuttle);
+newProjects.add(newProjectExpr);
+ }
+}
+
+final RelDataType newRowType =
+RelOptUtil.permute(calc.getCluster().getTypeFactory(), rowType,
+mapping);
+
+final RelNode newInputRelNode = relBuilder.push(newInput).build();
+if (rexProgram.getCondition() != null) {
+ final List filter = Lists.transform(
+ ImmutableList.of(
+ rexProgram.getCondition()), rexProgram::expandLocalRef);
+ assert filter.size() == 1;
+ final RexNode conditionExpr = filter.get(0);
+
+ final RexNode newConditionExpr =
+ conditionExpr.accept(shuttle);
+ final RexProgram newRexProgram =
RexProgram.create(newInputRelNode.getRowType(),
+ newProjects, newConditionExpr, newRowType.getFieldNames(),
+ newInputRelNode.getCluster().getRexBuilder());
+ LogicalCalc logicalCalc = LogicalCalc.create(newInputRelNode,
newRexProgram);
+
+ return result(logicalCalc, mapping);
+} else {
+ final RexProgram newRexProgram = RexProgram
+ .create(newInputRelNode.getRowType(), newProjects, null,
+ newRowType.getFieldNames(),
newInputRelNode.getCluster().getRexBuilder());
+
+ LogicalCalc logicalCalc = LogicalCalc.create(newInputRelNode,
newRexProgram);
+ return result(logicalCalc, mapping);
Review comment:
Hints need to be transferred from the original Calc to the new Calc (see
CALCITE-4055).
Also, IMHO this piece of code could be refactored, to avoid duplicating
instructions at the end of the "if" and "else" block, something like:
```
RexNode newConditionExpr = null;
if (rexProgram.getCondition() != null) {
...
newConditionExpr = conditionExpr.accept(shuttle);
}
final RexProgram newRexProgram = ...
final LogicalCalc logicalCalc = LogicalCalc.create(newInputRelNode,
newRexProgram);
logicalCalc.withHints(calc.getHints()); // transfer hints
return result(logicalCalc, mapping);
```
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [calcite] rubenada commented on pull request #1987: [CALCITE-4020] Support Calc operator in RelFieldTrimmer
rubenada commented on pull request #1987: URL: https://github.com/apache/calcite/pull/1987#issuecomment-643116159 @xy2953396112 Please bear in mind that [CALCITE-4055 - RelFieldTrimmer loses hints](https://issues.apache.org/jira/browse/CALCITE-4055) ([PR#2015](https://github.com/apache/calcite/pull/2015)) has been merged, which leads to some conflicts in the current PR. Sorry for the inconvenience. Also, since you are adding the processing of Calc, which implements Hintable, you should take into consideration the guidelines from CALCITE-4055 in order to transfer the hints from the original Calc to the trimmed Calc. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[calcite] branch master updated: [CALCITE-4055] RelFieldTrimmer loses hints
This is an automated email from the ASF dual-hosted git repository.
rubenql pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/calcite.git
The following commit(s) were added to refs/heads/master by this push:
new 7c2f677 [CALCITE-4055] RelFieldTrimmer loses hints
7c2f677 is described below
commit 7c2f6772ee6bbfe4fefc2cf35152d6c71cc7a361
Author: rubenada
AuthorDate: Tue Jun 9 15:11:34 2020 +0200
[CALCITE-4055] RelFieldTrimmer loses hints
---
.../apache/calcite/sql2rel/RelFieldTrimmer.java| 32 +++--
.../java/org/apache/calcite/tools/RelBuilder.java | 7 +-
.../calcite/sql2rel/RelFieldTrimmerTest.java | 137 +++--
3 files changed, 161 insertions(+), 15 deletions(-)
diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java
b/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java
index 2da00c0..0a4ec01 100644
--- a/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java
+++ b/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java
@@ -390,7 +390,7 @@ public class RelFieldTrimmer implements ReflectiveVisitor {
// Some parts of the system can't handle rows with zero fields, so
// pretend that one field is used.
if (fieldsUsed.cardinality() == 0) {
- return dummyProject(fieldCount, newInput);
+ return dummyProject(fieldCount, newInput, project);
}
// Build new project expressions, and populate the mapping.
@@ -417,7 +417,8 @@ public class RelFieldTrimmer implements ReflectiveVisitor {
relBuilder.push(newInput);
relBuilder.project(newProjects, newRowType.getFieldNames());
-return result(relBuilder.build(), mapping);
+final RelNode newProject = RelOptUtil.propagateRelHints(project,
relBuilder.build());
+return result(newProject, mapping);
}
/** Creates a project with a dummy column, to protect the parts of the system
@@ -425,9 +426,21 @@ public class RelFieldTrimmer implements ReflectiveVisitor {
*
* @param fieldCount Number of fields in the original relational expression
* @param input Trimmed input
- * @return Dummy project, or null if no dummy is required
+ * @return Dummy project
*/
protected TrimResult dummyProject(int fieldCount, RelNode input) {
+return dummyProject(fieldCount, input, null);
+ }
+
+ /** Creates a project with a dummy column, to protect the parts of the system
+ * that cannot handle a relational expression with no columns.
+ *
+ * @param fieldCount Number of fields in the original relational expression
+ * @param input Trimmed input
+ * @param originalRelNode Source RelNode for hint propagation (or null if no
propagation needed)
+ * @return Dummy project
+ */
+ protected TrimResult dummyProject(int fieldCount, RelNode input, RelNode
originalRelNode) {
final RelOptCluster cluster = input.getCluster();
final Mapping mapping =
Mappings.create(MappingType.INVERSE_SURJECTION, fieldCount, 1);
@@ -439,8 +452,12 @@ public class RelFieldTrimmer implements ReflectiveVisitor {
final RexLiteral expr =
cluster.getRexBuilder().makeExactLiteral(BigDecimal.ZERO);
relBuilder.push(input);
-relBuilder.project(ImmutableList.of(expr),
ImmutableList.of("DUMMY"));
-return result(relBuilder.build(), mapping);
+relBuilder.project(ImmutableList.of(expr), ImmutableList.of("DUMMY"));
+RelNode newProject = relBuilder.build();
+if (originalRelNode != null) {
+ newProject = RelOptUtil.propagateRelHints(originalRelNode, newProject);
+}
+return result(newProject, mapping);
}
/**
@@ -773,7 +790,7 @@ public class RelFieldTrimmer implements ReflectiveVisitor {
default:
relBuilder.join(join.getJoinType(), newConditionExpr);
}
-
+relBuilder.hints(join.getHints());
return result(relBuilder.build(), mapping);
}
@@ -971,7 +988,8 @@ public class RelFieldTrimmer implements ReflectiveVisitor {
(Iterable) newGroupSets);
relBuilder.aggregate(groupKey, newAggCallList);
-return result(relBuilder.build(), mapping);
+final RelNode newAggregate = RelOptUtil.propagateRelHints(aggregate,
relBuilder.build());
+return result(newAggregate, mapping);
}
/**
diff --git a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java
b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java
index 77e10c0..3362a24 100644
--- a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java
+++ b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java
@@ -2718,11 +2718,16 @@ public class RelBuilder {
*/
public RelBuilder hints(Iterable hints) {
Objects.requireNonNull(hints);
+final List relHintList = hints instanceof List ? (List)
hints
+: Lists.newArrayList(hints);
+if (relHintList.isEmpty()) {
+ return this;
+}
final Frame frame = peek_();
assert frame != null : "There is no relational expression to
[GitHub] [calcite] rubenada merged pull request #2015: [CALCITE-4055] RelFieldTrimmer loses hints
rubenada merged pull request #2015: URL: https://github.com/apache/calcite/pull/2015 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
