This is an automated email from the ASF dual-hosted git repository.
jiajunxie pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/calcite.git
The following commit(s) were added to refs/heads/main by this push:
new d68656a251 [CALCITE-5657] Add ARRAY_DISTINCT function (enabled in
Spark library).
d68656a251 is described below
commit d68656a251b08c923101d881ebeea37601887229
Author: yongen.ly <[email protected]>
AuthorDate: Wed Apr 19 00:26:04 2023 +0800
[CALCITE-5657] Add ARRAY_DISTINCT function (enabled in Spark library).
---
.../org/apache/calcite/adapter/enumerable/RexImpTable.java | 2 ++
.../main/java/org/apache/calcite/runtime/SqlFunctions.java | 7 +++++++
core/src/main/java/org/apache/calcite/sql/SqlKind.java | 3 +++
.../java/org/apache/calcite/sql/fun/SqlLibraryOperators.java | 7 +++++++
.../src/main/java/org/apache/calcite/util/BuiltInMethod.java | 1 +
site/_docs/reference.md | 1 +
.../main/java/org/apache/calcite/test/SqlOperatorTest.java | 12 ++++++++++++
7 files changed, 33 insertions(+)
diff --git
a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java
b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java
index 9ad452277c..3c2ecf8edc 100644
--- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java
+++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java
@@ -115,6 +115,7 @@ import static
org.apache.calcite.sql.fun.SqlInternalOperators.THROW_UNLESS;
import static org.apache.calcite.sql.fun.SqlLibraryOperators.ARRAY_AGG;
import static org.apache.calcite.sql.fun.SqlLibraryOperators.ARRAY_CONCAT;
import static org.apache.calcite.sql.fun.SqlLibraryOperators.ARRAY_CONCAT_AGG;
+import static org.apache.calcite.sql.fun.SqlLibraryOperators.ARRAY_DISTINCT;
import static org.apache.calcite.sql.fun.SqlLibraryOperators.ARRAY_LENGTH;
import static org.apache.calcite.sql.fun.SqlLibraryOperators.ARRAY_REVERSE;
import static org.apache.calcite.sql.fun.SqlLibraryOperators.BOOL_AND;
@@ -668,6 +669,7 @@ public class RexImpTable {
defineMethod(ELEMENT, BuiltInMethod.ELEMENT.method, NullPolicy.STRICT);
defineMethod(STRUCT_ACCESS, BuiltInMethod.STRUCT_ACCESS.method,
NullPolicy.ANY);
defineMethod(MEMBER_OF, BuiltInMethod.MEMBER_OF.method, NullPolicy.NONE);
+ defineMethod(ARRAY_DISTINCT, BuiltInMethod.ARRAY_DISTINCT.method,
NullPolicy.STRICT);
defineMethod(ARRAY_REVERSE, BuiltInMethod.ARRAY_REVERSE.method,
NullPolicy.STRICT);
map.put(ARRAY_CONCAT, new ArrayConcatImplementor());
final MethodImplementor isEmptyImplementor =
diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java
b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java
index 83a511ff12..75ced53c9a 100644
--- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java
+++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java
@@ -92,6 +92,7 @@ import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
@@ -3790,6 +3791,12 @@ public class SqlFunctions {
return atomic;
}
+ /** Support the ARRAY_DISTINCT function. */
+ public static List distinct(List list) {
+ Set result = new LinkedHashSet<>(list);
+ return new ArrayList<>(result);
+ }
+
/** Support the SLICE function. */
public static List slice(List list) {
List result = new ArrayList(list.size());
diff --git a/core/src/main/java/org/apache/calcite/sql/SqlKind.java
b/core/src/main/java/org/apache/calcite/sql/SqlKind.java
index 287b7d1a47..44cd4cd02d 100644
--- a/core/src/main/java/org/apache/calcite/sql/SqlKind.java
+++ b/core/src/main/java/org/apache/calcite/sql/SqlKind.java
@@ -680,6 +680,9 @@ public enum SqlKind {
/** {@code ARRAY_CONCAT} function (BigQuery semantics). */
ARRAY_CONCAT,
+ /** {@code ARRAY_DISTINCT} function (Spark semantics). */
+ ARRAY_DISTINCT,
+
/** {@code ARRAY_REVERSE} function (BigQuery semantics). */
ARRAY_REVERSE,
diff --git
a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java
b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java
index b75dcab0fb..a2f1254147 100644
--- a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java
+++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java
@@ -825,6 +825,13 @@ public abstract class SqlLibraryOperators {
.withOperandTypeInference(InferTypes.RETURN_TYPE)
.withKind(SqlKind.CONCAT2);
+ /** The "ARRAY_DISTINCT(array)" function. */
+ @LibraryOperator(libraries = {SPARK})
+ public static final SqlFunction ARRAY_DISTINCT =
+ SqlBasicFunction.create(SqlKind.ARRAY_DISTINCT,
+ ReturnTypes.ARG0_NULLABLE,
+ OperandTypes.ARRAY);
+
/** The "ARRAY_LENGTH(array)" function. */
@LibraryOperator(libraries = {BIG_QUERY})
public static final SqlFunction ARRAY_LENGTH =
diff --git a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java
b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java
index 6b02218ec8..90eb4df0fc 100644
--- a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java
+++ b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java
@@ -628,6 +628,7 @@ public enum BuiltInMethod {
IS_EMPTY(Collection.class, "isEmpty"),
SUBMULTISET_OF(SqlFunctions.class, "submultisetOf", Collection.class,
Collection.class),
+ ARRAY_DISTINCT(SqlFunctions.class, "distinct", List.class),
ARRAY_REVERSE(SqlFunctions.class, "reverse", List.class),
SELECTIVITY(Selectivity.class, "getSelectivity", RexNode.class),
UNIQUE_KEYS(UniqueKeys.class, "getUniqueKeys", boolean.class),
diff --git a/site/_docs/reference.md b/site/_docs/reference.md
index 52d8bfcb35..d5d58c3568 100644
--- a/site/_docs/reference.md
+++ b/site/_docs/reference.md
@@ -2651,6 +2651,7 @@ BigQuery's type system uses confusingly different names
for types and functions:
| p | expr :: type | Casts *expr* to *type*
| m | expr1 <=> expr2 | Whether two values are
equal, treating null values as the same, and it's similar to `IS NOT DISTINCT
FROM`
| b | ARRAY_CONCAT(array [, array ]*) | Concatenates one or
more arrays. If any input argument is `NULL` the function returns `NULL`
+| s | ARRAY_DISTINCT(array) | Returns unique elements
of *array*. Keeps ordering of elements.
| b | ARRAY_LENGTH(array) | Synonym for
`CARDINALITY`
| b | ARRAY_REVERSE(array) | Reverses elements of
*array*
| m s | CHAR(integer) | Returns the character
whose ASCII code is *integer* % 256, or null if *integer* < 0
diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java
b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java
index 9339bf2d95..694f026c01 100644
--- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java
+++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java
@@ -5324,6 +5324,18 @@ public class SqlOperatorTest {
f.checkNull("array_concat(cast(null as integer array), array[1])");
}
+ /** Tests {@code ARRAY_DISTINCT} function from Spark. */
+ @Test void testArrayDistinctFunc() {
+ SqlOperatorFixture f = fixture()
+ .setFor(SqlLibraryOperators.ARRAY_DISTINCT)
+ .withLibrary(SqlLibrary.SPARK);
+ f.checkScalar("array_distinct(array[1, 2, 2, 1])", "[1, 2]",
+ "INTEGER NOT NULL ARRAY NOT NULL");
+ f.checkScalar("array_distinct(array[null, 1, null])", "[null, 1]",
+ "INTEGER ARRAY NOT NULL");
+ f.checkNull("array_distinct(null)");
+ }
+
/** Tests {@code ARRAY_REVERSE} function from BigQuery. */
@Test void testArrayReverseFunc() {
SqlOperatorFixture f = fixture()
