This is an automated email from the ASF dual-hosted git repository. jiajunxie pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/calcite.git
The following commit(s) were added to refs/heads/main by this push: new d68656a251 [CALCITE-5657] Add ARRAY_DISTINCT function (enabled in Spark library). d68656a251 is described below commit d68656a251b08c923101d881ebeea37601887229 Author: yongen.ly <yongen...@alibaba-inc.com> AuthorDate: Wed Apr 19 00:26:04 2023 +0800 [CALCITE-5657] Add ARRAY_DISTINCT function (enabled in Spark library). --- .../org/apache/calcite/adapter/enumerable/RexImpTable.java | 2 ++ .../main/java/org/apache/calcite/runtime/SqlFunctions.java | 7 +++++++ core/src/main/java/org/apache/calcite/sql/SqlKind.java | 3 +++ .../java/org/apache/calcite/sql/fun/SqlLibraryOperators.java | 7 +++++++ .../src/main/java/org/apache/calcite/util/BuiltInMethod.java | 1 + site/_docs/reference.md | 1 + .../main/java/org/apache/calcite/test/SqlOperatorTest.java | 12 ++++++++++++ 7 files changed, 33 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java index 9ad452277c..3c2ecf8edc 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java @@ -115,6 +115,7 @@ import static org.apache.calcite.sql.fun.SqlInternalOperators.THROW_UNLESS; import static org.apache.calcite.sql.fun.SqlLibraryOperators.ARRAY_AGG; import static org.apache.calcite.sql.fun.SqlLibraryOperators.ARRAY_CONCAT; import static org.apache.calcite.sql.fun.SqlLibraryOperators.ARRAY_CONCAT_AGG; +import static org.apache.calcite.sql.fun.SqlLibraryOperators.ARRAY_DISTINCT; import static org.apache.calcite.sql.fun.SqlLibraryOperators.ARRAY_LENGTH; import static org.apache.calcite.sql.fun.SqlLibraryOperators.ARRAY_REVERSE; import static org.apache.calcite.sql.fun.SqlLibraryOperators.BOOL_AND; @@ -668,6 +669,7 @@ public class RexImpTable { defineMethod(ELEMENT, BuiltInMethod.ELEMENT.method, NullPolicy.STRICT); defineMethod(STRUCT_ACCESS, BuiltInMethod.STRUCT_ACCESS.method, NullPolicy.ANY); defineMethod(MEMBER_OF, BuiltInMethod.MEMBER_OF.method, NullPolicy.NONE); + defineMethod(ARRAY_DISTINCT, BuiltInMethod.ARRAY_DISTINCT.method, NullPolicy.STRICT); defineMethod(ARRAY_REVERSE, BuiltInMethod.ARRAY_REVERSE.method, NullPolicy.STRICT); map.put(ARRAY_CONCAT, new ArrayConcatImplementor()); final MethodImplementor isEmptyImplementor = diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java index 83a511ff12..75ced53c9a 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java @@ -92,6 +92,7 @@ import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; import java.util.Locale; @@ -3790,6 +3791,12 @@ public class SqlFunctions { return atomic; } + /** Support the ARRAY_DISTINCT function. */ + public static List distinct(List list) { + Set result = new LinkedHashSet<>(list); + return new ArrayList<>(result); + } + /** Support the SLICE function. */ public static List slice(List list) { List result = new ArrayList(list.size()); diff --git a/core/src/main/java/org/apache/calcite/sql/SqlKind.java b/core/src/main/java/org/apache/calcite/sql/SqlKind.java index 287b7d1a47..44cd4cd02d 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlKind.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlKind.java @@ -680,6 +680,9 @@ public enum SqlKind { /** {@code ARRAY_CONCAT} function (BigQuery semantics). */ ARRAY_CONCAT, + /** {@code ARRAY_DISTINCT} function (Spark semantics). */ + ARRAY_DISTINCT, + /** {@code ARRAY_REVERSE} function (BigQuery semantics). */ ARRAY_REVERSE, diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java index b75dcab0fb..a2f1254147 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java @@ -825,6 +825,13 @@ public abstract class SqlLibraryOperators { .withOperandTypeInference(InferTypes.RETURN_TYPE) .withKind(SqlKind.CONCAT2); + /** The "ARRAY_DISTINCT(array)" function. */ + @LibraryOperator(libraries = {SPARK}) + public static final SqlFunction ARRAY_DISTINCT = + SqlBasicFunction.create(SqlKind.ARRAY_DISTINCT, + ReturnTypes.ARG0_NULLABLE, + OperandTypes.ARRAY); + /** The "ARRAY_LENGTH(array)" function. */ @LibraryOperator(libraries = {BIG_QUERY}) public static final SqlFunction ARRAY_LENGTH = diff --git a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java index 6b02218ec8..90eb4df0fc 100644 --- a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java +++ b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java @@ -628,6 +628,7 @@ public enum BuiltInMethod { IS_EMPTY(Collection.class, "isEmpty"), SUBMULTISET_OF(SqlFunctions.class, "submultisetOf", Collection.class, Collection.class), + ARRAY_DISTINCT(SqlFunctions.class, "distinct", List.class), ARRAY_REVERSE(SqlFunctions.class, "reverse", List.class), SELECTIVITY(Selectivity.class, "getSelectivity", RexNode.class), UNIQUE_KEYS(UniqueKeys.class, "getUniqueKeys", boolean.class), diff --git a/site/_docs/reference.md b/site/_docs/reference.md index 52d8bfcb35..d5d58c3568 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -2651,6 +2651,7 @@ BigQuery's type system uses confusingly different names for types and functions: | p | expr :: type | Casts *expr* to *type* | m | expr1 <=> expr2 | Whether two values are equal, treating null values as the same, and it's similar to `IS NOT DISTINCT FROM` | b | ARRAY_CONCAT(array [, array ]*) | Concatenates one or more arrays. If any input argument is `NULL` the function returns `NULL` +| s | ARRAY_DISTINCT(array) | Returns unique elements of *array*. Keeps ordering of elements. | b | ARRAY_LENGTH(array) | Synonym for `CARDINALITY` | b | ARRAY_REVERSE(array) | Reverses elements of *array* | m s | CHAR(integer) | Returns the character whose ASCII code is *integer* % 256, or null if *integer* < 0 diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index 9339bf2d95..694f026c01 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -5324,6 +5324,18 @@ public class SqlOperatorTest { f.checkNull("array_concat(cast(null as integer array), array[1])"); } + /** Tests {@code ARRAY_DISTINCT} function from Spark. */ + @Test void testArrayDistinctFunc() { + SqlOperatorFixture f = fixture() + .setFor(SqlLibraryOperators.ARRAY_DISTINCT) + .withLibrary(SqlLibrary.SPARK); + f.checkScalar("array_distinct(array[1, 2, 2, 1])", "[1, 2]", + "INTEGER NOT NULL ARRAY NOT NULL"); + f.checkScalar("array_distinct(array[null, 1, null])", "[null, 1]", + "INTEGER ARRAY NOT NULL"); + f.checkNull("array_distinct(null)"); + } + /** Tests {@code ARRAY_REVERSE} function from BigQuery. */ @Test void testArrayReverseFunc() { SqlOperatorFixture f = fixture()