This is an automated email from the ASF dual-hosted git repository. jhyde pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/calcite.git
commit 671f49d7d0ba09559a7d4977ebf250259a6fe526 Author: yongen.ly <yongen...@alibaba-inc.com> AuthorDate: Fri May 19 17:37:09 2023 +0800 [CALCITE-5714] Add MAP_ENTRIES function (enabled in Spark library) --- .../calcite/adapter/enumerable/RexImpTable.java | 2 ++ .../org/apache/calcite/runtime/SqlFunctions.java | 9 +++++++++ .../main/java/org/apache/calcite/sql/SqlKind.java | 3 +++ .../apache/calcite/sql/fun/SqlLibraryOperators.java | 7 +++++++ .../org/apache/calcite/sql/type/ReturnTypes.java | 21 +++++++++++++++++++++ .../apache/calcite/sql/type/SqlTypeTransforms.java | 11 +++++++++++ .../org/apache/calcite/sql/type/SqlTypeUtil.java | 10 ++++++++++ .../java/org/apache/calcite/util/BuiltInMethod.java | 1 + site/_docs/reference.md | 1 + .../org/apache/calcite/test/SqlOperatorTest.java | 14 ++++++++++++++ 10 files changed, 79 insertions(+) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java index 620926cf34..925bbec92e 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java @@ -172,6 +172,7 @@ import static org.apache.calcite.sql.fun.SqlLibraryOperators.LOG; import static org.apache.calcite.sql.fun.SqlLibraryOperators.LOGICAL_AND; import static org.apache.calcite.sql.fun.SqlLibraryOperators.LOGICAL_OR; import static org.apache.calcite.sql.fun.SqlLibraryOperators.LPAD; +import static org.apache.calcite.sql.fun.SqlLibraryOperators.MAP_ENTRIES; import static org.apache.calcite.sql.fun.SqlLibraryOperators.MAP_KEYS; import static org.apache.calcite.sql.fun.SqlLibraryOperators.MAP_VALUES; import static org.apache.calcite.sql.fun.SqlLibraryOperators.MAX_BY; @@ -693,6 +694,7 @@ public class RexImpTable { defineMethod(ARRAY_REPEAT, BuiltInMethod.ARRAY_REPEAT.method, NullPolicy.NONE); defineMethod(ARRAY_REVERSE, BuiltInMethod.ARRAY_REVERSE.method, NullPolicy.STRICT); defineMethod(ARRAY_SIZE, BuiltInMethod.COLLECTION_SIZE.method, NullPolicy.STRICT); + defineMethod(MAP_ENTRIES, BuiltInMethod.MAP_ENTRIES.method, NullPolicy.STRICT); defineMethod(MAP_KEYS, BuiltInMethod.MAP_KEYS.method, NullPolicy.STRICT); defineMethod(MAP_VALUES, BuiltInMethod.MAP_VALUES.method, NullPolicy.STRICT); map.put(ARRAY_CONCAT, new ArrayConcatImplementor()); diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java index e47a2d86f9..8c68768d1b 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java @@ -3898,6 +3898,15 @@ public class SqlFunctions { return Collections.nCopies(numberOfElement, element); } + /** Support the MAP_ENTRIES function. */ + public static List mapEntries(Map<Object, Object> map) { + final List result = new ArrayList(map.size()); + for (Map.Entry<Object, Object> entry : map.entrySet()) { + result.add(Arrays.asList(entry.getKey(), entry.getValue())); + } + return result; + } + /** Support the MAP_KEYS function. */ public static List mapKeys(Map map) { return new ArrayList<>(map.keySet()); diff --git a/core/src/main/java/org/apache/calcite/sql/SqlKind.java b/core/src/main/java/org/apache/calcite/sql/SqlKind.java index f8e10a41f4..b1f18b7136 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlKind.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlKind.java @@ -695,6 +695,9 @@ public enum SqlKind { /** {@code ARRAY_SIZE} function (Spark semantics). */ ARRAY_SIZE, + /** {@code MAP_ENTRIES} function (Spark semantics). */ + MAP_ENTRIES, + /** {@code MAP_KEYS} function (Spark semantics). */ MAP_KEYS, diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java index 476024dad5..bcc6e3fa1f 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java @@ -923,6 +923,13 @@ public abstract class SqlLibraryOperators { ReturnTypes.LEAST_RESTRICTIVE, OperandTypes.AT_LEAST_ONE_SAME_VARIADIC); + /** The "MAP_ENTRIES(map)" function. */ + @LibraryOperator(libraries = {SPARK}) + public static final SqlFunction MAP_ENTRIES = + SqlBasicFunction.create(SqlKind.MAP_ENTRIES, + ReturnTypes.TO_MAP_ENTRIES_NULLABLE, + OperandTypes.MAP); + /** The "MAP_KEYS(map)" function. */ @LibraryOperator(libraries = {SPARK}) public static final SqlFunction MAP_KEYS = diff --git a/core/src/main/java/org/apache/calcite/sql/type/ReturnTypes.java b/core/src/main/java/org/apache/calcite/sql/type/ReturnTypes.java index 09d23db838..adf1ca3902 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/ReturnTypes.java +++ b/core/src/main/java/org/apache/calcite/sql/type/ReturnTypes.java @@ -536,6 +536,27 @@ public abstract class ReturnTypes { public static final SqlReturnTypeInference TO_MAP = ARG0.andThen(SqlTypeTransforms.TO_MAP); + /** + * Returns a ROW type. + * + * <p>For example, given {@code (INTEGER, DATE) MAP}, returns + * {@code Record(f0: INTEGER, f1: DATE)}. + */ + public static final SqlReturnTypeInference TO_ROW = + ARG0.andThen(SqlTypeTransforms.TO_ROW); + + /** + * Returns a ARRAY type. + * + * <p>For example, given {@code (INTEGER, DATE) MAP}, returns + * {@code Record(f0: INTEGER, f1: DATE) ARRAY}. + */ + public static final SqlReturnTypeInference TO_MAP_ENTRIES = + TO_ROW.andThen(SqlTypeTransforms.TO_ARRAY); + + public static final SqlReturnTypeInference TO_MAP_ENTRIES_NULLABLE = + TO_MAP_ENTRIES.andThen(SqlTypeTransforms.TO_NULLABLE); + /** * Returns a ARRAY type. * diff --git a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeTransforms.java b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeTransforms.java index f87b6992f0..4adb1133c5 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeTransforms.java +++ b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeTransforms.java @@ -228,6 +228,17 @@ public abstract class SqlTypeTransforms { SqlTypeUtil.createMapTypeFromRecord(opBinding.getTypeFactory(), typeToTransform); + /** + * Parameter type-inference transform strategy that converts a MAP type + * to a two-field record type. + * + * @see org.apache.calcite.rel.type.RelDataTypeFactory#createStructType + */ + public static final SqlTypeTransform TO_ROW = + (opBinding, typeToTransform) -> + SqlTypeUtil.createRecordTypeFromMap(opBinding.getTypeFactory(), + typeToTransform); + /** * Parameter type-inference transform strategy that converts a MAP type * to a ARRAY type. diff --git a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java index 2fdd95b34e..5503a2034d 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java +++ b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java @@ -54,6 +54,7 @@ import java.math.BigDecimal; import java.nio.charset.Charset; import java.util.AbstractList; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.Map; @@ -1197,6 +1198,15 @@ public abstract class SqlTypeUtil { type.getFieldList().get(1).getType(), false); } + /** Creates a ROW type from a map type. The record type will have two fields. */ + public static RelDataType createRecordTypeFromMap( + RelDataTypeFactory typeFactory, RelDataType type) { + RelDataType keyType = requireNonNull(type.getKeyType(), () -> "keyType of " + type); + RelDataType valueType = requireNonNull(type.getValueType(), () -> "keyType of " + type); + return typeFactory.createStructType( + Arrays.asList(keyType, valueType), Arrays.asList("f0", "f1")); + } + /** * Adds collation and charset to a character type, returns other types * unchanged. diff --git a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java index 415f965c2e..ff13006a22 100644 --- a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java +++ b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java @@ -632,6 +632,7 @@ public enum BuiltInMethod { ARRAY_DISTINCT(SqlFunctions.class, "distinct", List.class), ARRAY_REPEAT(SqlFunctions.class, "repeat", Object.class, Integer.class), ARRAY_REVERSE(SqlFunctions.class, "reverse", List.class), + MAP_ENTRIES(SqlFunctions.class, "mapEntries", Map.class), MAP_KEYS(SqlFunctions.class, "mapKeys", Map.class), MAP_VALUES(SqlFunctions.class, "mapValues", Map.class), SELECTIVITY(Selectivity.class, "getSelectivity", RexNode.class), diff --git a/site/_docs/reference.md b/site/_docs/reference.md index 090dd07226..cc143f2ee2 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -2731,6 +2731,7 @@ BigQuery's type system uses confusingly different names for types and functions: | m | TO_BASE64(string) | Converts the *string* to base-64 encoded form and returns a encoded string | b m | FROM_BASE64(string) | Returns the decoded result of a base-64 *string* as a string | b o | LTRIM(string) | Returns *string* with all blanks removed from the start +| s | MAP_ENTRIES(map) | Returns the entries of the *map* as an array, the order of the entries is not defined | s | MAP_KEYS(map) | Returns the keys of the *map* as an array, the order of the entries is not defined | s | MAP_VALUES(map) | Returns the values of the *map* as an array, the order of the entries is not defined | b m p | MD5(string) | Calculates an MD5 128-bit checksum of *string* and returns it as a hex string diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index 9097ace048..aa3ae30eeb 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -5437,6 +5437,20 @@ public class SqlOperatorTest { f.checkNull("array_length(null)"); } + /** Tests {@code MAP_ENTRIES} function from Spark. */ + @Test void testMapEntriesFunc() { + final SqlOperatorFixture f0 = fixture(); + f0.setFor(SqlLibraryOperators.MAP_ENTRIES); + f0.checkFails("^map_entries(map['foo', 1, 'bar', 2])^", + "No match found for function signature MAP_ENTRIES\\(<\\(CHAR\\(3\\), INTEGER\\) " + + "MAP>\\)", false); + final SqlOperatorFixture f = f0.withLibrary(SqlLibrary.SPARK); + f.checkScalar("map_entries(map['foo', 1, 'bar', 2])", "[{foo, 1}, {bar, 2}]", + "RecordType(CHAR(3) NOT NULL f0, INTEGER NOT NULL f1) NOT NULL ARRAY NOT NULL"); + f.checkScalar("map_entries(map['foo', 1, null, 2])", "[{foo, 1}, {null, 2}]", + "RecordType(CHAR(3) f0, INTEGER NOT NULL f1) NOT NULL ARRAY NOT NULL"); + } + /** Tests {@code MAP_KEYS} function from Spark. */ @Test void testMapKeysFunc() { final SqlOperatorFixture f0 = fixture();