This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/dev/pr-12636-53d3e7c152dd125a01b36fe955eeb82129661234 in repository https://gitbox.apache.org/repos/asf/seatunnel.git
commit 6ab102d56cffc92b29773d2733c0f55e07ff8e08 Author: hutiefang76 <[email protected]> AuthorDate: Fri Oct 9 03:25:29 2026 +0000 [Fix][Transform-V2] Handle signed collection expressions (#12636) --- .../introduction/concepts/incompatible-changes.md | 4 ++ docs/en/transforms/sql.md | 6 +++ .../introduction/concepts/incompatible-changes.md | 4 ++ docs/zh/transforms/sql.md | 6 +++ .../test/resources/sql_transform/nested_type.conf | 5 +- .../transform/sql/zeta/ZetaSQLFunction.java | 19 ++++++- .../sql/zeta/functions/ArrayFunction.java | 6 +-- .../sql/zeta/functions/CommonFunction.java | 9 ++++ .../seatunnel/transform/sql/SQLNestedTypeTest.java | 58 ++++++++++++++++++++++ 9 files changed, 111 insertions(+), 6 deletions(-) diff --git a/docs/en/introduction/concepts/incompatible-changes.md b/docs/en/introduction/concepts/incompatible-changes.md index 340ebb8492..b89ac06de8 100644 --- a/docs/en/introduction/concepts/incompatible-changes.md +++ b/docs/en/introduction/concepts/incompatible-changes.md @@ -5,6 +5,10 @@ You need to check this document before you upgrade to related version. ## dev +### SQL TINYINT array schema + +The Zeta SQL ARRAY function now declares TINYINT elements as `ARRAY<TINYINT>`, matching the Byte values it emits. The previous `ARRAY<STRING>` declaration could fail in schema-dependent row consumers. Update downstream declarations that assumed STRING elements; cast the SQL values to STRING explicitly when that schema is required. Restart affected jobs with the corrected schema rather than restoring state that relies on the old declaration. + ### DuckDB BIT and ENUM automatic DDL - Scalar `BIT` and `ENUM` columns with no catalog length now retain an unspecified STRING length diff --git a/docs/en/transforms/sql.md b/docs/en/transforms/sql.md index 3d998b43e5..6b6691b219 100644 --- a/docs/en/transforms/sql.md +++ b/docs/en/transforms/sql.md @@ -155,6 +155,12 @@ sink { } ``` +## Collection expressions + +ARRAY and MAP arguments may use unary `+`/`-` and parentheses. For example, `ARRAY(-1, +2, (3))` produces `[-1, 2, 3]`, and `MAP('negative', (-1))` produces a map with value `-1`. + +Signed collection expressions retain the numeric element type and propagate NULL operands. TINYINT arrays now use `ARRAY<TINYINT>` rather than the previous incorrect `ARRAY<STRING>` schema. + ## Changelog - Support struct query diff --git a/docs/zh/introduction/concepts/incompatible-changes.md b/docs/zh/introduction/concepts/incompatible-changes.md index 7facd5f3c2..002cd6642e 100644 --- a/docs/zh/introduction/concepts/incompatible-changes.md +++ b/docs/zh/introduction/concepts/incompatible-changes.md @@ -4,6 +4,10 @@ ## dev +### SQL TINYINT 数组模式 + +Zeta SQL ARRAY 函数现在将 TINYINT 元素声明为 `ARRAY<TINYINT>`，与实际输出的 Byte 值一致。此前错误的 `ARRAY<STRING>` 声明会导致依赖模式的行处理失败。请更新假定元素为 STRING 的下游声明；需要字符串模式时，在 SQL 中显式将值转换为 STRING。受影响的作业应使用修正后的模式重新启动，不要恢复依赖旧声明的状态。 + ### DuckDB BIT 和 ENUM 自动建表 - Catalog 未提供长度时，标量 `BIT` 和 `ENUM` 列现在保留未指定的 STRING 长度，不再使用原来的 1/255 回退值。 diff --git a/docs/zh/transforms/sql.md b/docs/zh/transforms/sql.md index 754c37f006..4e6596ed53 100644 --- a/docs/zh/transforms/sql.md +++ b/docs/zh/transforms/sql.md @@ -155,6 +155,12 @@ sink { } ``` +## 集合表达式 + +ARRAY 和 MAP 的参数支持一元 `+`/`-` 和括号。例如，`ARRAY(-1, +2, (3))` 生成 `[-1, 2, 3]`，`MAP('negative', (-1))` 生成值为 `-1` 的映射。 + +带符号的集合表达式保留数值元素类型，并传递 NULL 操作数。TINYINT 数组现在使用 `ARRAY<TINYINT>`，替代此前错误的 `ARRAY<STRING>` 模式。 + ## 更新日志 ### 新版本 diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/sql_transform/nested_type.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/sql_transform/nested_type.conf index 7db99cd4c5..9b83b1f104 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/sql_transform/nested_type.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/sql_transform/nested_type.conf @@ -44,7 +44,10 @@ transform { ARRAY(ARRAY(ARRAY(1,2,3), ARRAY(4,5,6), ARRAY(ARRAY(1,2,3), ARRAY(4,5,6)), ARRAY(3, 4))) as arr_of_arr, MAP('k', MAP('k', MAP('k', 1))) as map_of_map, ARRAY(MAP('k', 1), MAP('k2', ARRAY(1, 2))) as arr_of_map, - MAP('k', ARRAY(1, 2)) as map_of_arr + MAP('k', ARRAY(1, 2)) as map_of_arr, + ARRAY(-1, +2, (3)) as signed_array, + MAP('negative', (-1)) as signed_map, + ARRAY((MAP('k', -2))) as wrapped_map_array from dual """ } diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLFunction.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLFunction.java index ae10a9dd92..233245d980 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLFunction.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLFunction.java @@ -283,6 +283,21 @@ public class ZetaSQLFunction { SignedExpression signedExpression = (SignedExpression) expression; if (signedExpression.getSign() == '-') { Object value = computeForValue(signedExpression.getExpression(), inputFields); + if (value == null) { + return null; + } + if (value instanceof Byte) { + return (byte) -((Byte) value); + } + if (value instanceof Short) { + return (short) -((Short) value); + } + if (value instanceof Float) { + return -((Float) value); + } + if (value instanceof BigDecimal) { + return ((BigDecimal) value).negate(); + } if (value instanceof Integer) { return -((Integer) value); } @@ -295,8 +310,8 @@ public class ZetaSQLFunction { if (value instanceof Number) { return -((Number) value).doubleValue(); } - } else { - return computeForValue(signedExpression, inputFields); + } else if (signedExpression.getSign() == '+') { + return computeForValue(signedExpression.getExpression(), inputFields); } } if (expression instanceof DoubleValue) { diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/ArrayFunction.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/ArrayFunction.java index 1fdaecf05f..75f7dc9407 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/ArrayFunction.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/ArrayFunction.java @@ -134,8 +134,7 @@ public class ArrayFunction { } static ArrayType createArrayType(SeaTunnelDataType<?> elementType) { - if (elementType == BasicType.BYTE_TYPE || elementType == BasicType.VOID_TYPE) - return ArrayType.STRING_ARRAY_TYPE; + if (elementType == BasicType.VOID_TYPE) return ArrayType.STRING_ARRAY_TYPE; return ArrayType.of(elementType); } @@ -153,7 +152,8 @@ public class ArrayFunction { } private static boolean isNumericType(Class<?> type) { - return type == Short.class + return type == Byte.class + || type == Short.class || type == Integer.class || type == Long.class || type == Float.class diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/CommonFunction.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/CommonFunction.java index 132e1b4c7a..a918e9742c 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/CommonFunction.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/CommonFunction.java @@ -29,6 +29,8 @@ import net.sf.jsqlparser.expression.Expression; import net.sf.jsqlparser.expression.Function; import net.sf.jsqlparser.expression.LongValue; import net.sf.jsqlparser.expression.NullValue; +import net.sf.jsqlparser.expression.Parenthesis; +import net.sf.jsqlparser.expression.SignedExpression; import net.sf.jsqlparser.expression.StringValue; import net.sf.jsqlparser.expression.operators.relational.ExpressionList; import net.sf.jsqlparser.schema.Column; @@ -41,6 +43,13 @@ public class CommonFunction { public static SeaTunnelDataType resolveExpressionType( Expression expression, SeaTunnelRowType rowType) { + // Signs and parentheses preserve the inner expression's data type. + if (expression instanceof SignedExpression) { + return resolveExpressionType(((SignedExpression) expression).getExpression(), rowType); + } + if (expression instanceof Parenthesis) { + return resolveExpressionType(((Parenthesis) expression).getExpression(), rowType); + } if (expression instanceof NullValue) { return null; } diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/SQLNestedTypeTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/SQLNestedTypeTest.java index ca0a78a415..c68bc6bba9 100644 --- a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/SQLNestedTypeTest.java +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/SQLNestedTypeTest.java @@ -34,6 +34,7 @@ import net.sf.jsqlparser.expression.LongValue; import net.sf.jsqlparser.expression.StringValue; import net.sf.jsqlparser.expression.operators.relational.ExpressionList; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -122,6 +123,63 @@ public class SQLNestedTypeTest { type); } + @Test + void testSignedAndParenthesizedCollectionExpressions() { + SQLEngine sql = zetaEngine(); + SeaTunnelRowType inType = dummyInputType(); + sql.init( + "test", + null, + inType, + "select ARRAY(-1, +2, (3)) as a, MAP('negative', (-1)) as m, " + + "ARRAY((MAP('k', -2))) as nested from test"); + SeaTunnelRowType outType = sql.typeMapping(new ArrayList<>()); + Assertions.assertEquals(ArrayType.INT_ARRAY_TYPE, outType.getFieldType(0)); + Assertions.assertEquals( + new MapType<>(BasicType.STRING_TYPE, BasicType.INT_TYPE), outType.getFieldType(1)); + List<SeaTunnelRow> out = sql.transformBySQL(dummyRow(), outType); + Assertions.assertArrayEquals(new Integer[] {-1, 2, 3}, (Object[]) out.get(0).getField(0)); + Assertions.assertEquals(Collections.singletonMap("negative", -1), out.get(0).getField(1)); + Assertions.assertEquals( + Collections.singletonMap("k", -2), ((Object[]) out.get(0).getField(2))[0]); + } + + @Test + void testSignedCollectionColumnsKeepTypesAndNullValues() { + SeaTunnelRowType inType = + new SeaTunnelRowType( + new String[] {"f", "s", "b", "i"}, + new SeaTunnelDataType[] { + BasicType.FLOAT_TYPE, + BasicType.SHORT_TYPE, + BasicType.BYTE_TYPE, + BasicType.INT_TYPE + }); + SQLEngine sql = zetaEngine(); + sql.init( + "test", + null, + inType, + "select ARRAY(-f) as a, MAP('k', -s) as m, ARRAY(-b) as b, MAP('null', -i) as nm, ARRAY((-s)) as sa, ARRAY(-b, +1) as mixed from test"); + SeaTunnelRowType outType = sql.typeMapping(new ArrayList<>()); + SeaTunnelRow out = + sql.transformBySQL( + new SeaTunnelRow(new Object[] {1.25f, (short) 2, (byte) 3, null}), + outType) + .get(0); + Assertions.assertEquals(ArrayType.FLOAT_ARRAY_TYPE, outType.getFieldType(0)); + Assertions.assertArrayEquals(new Float[] {-1.25f}, (Object[]) out.getField(0)); + Assertions.assertEquals(Collections.singletonMap("k", (short) -2), out.getField(1)); + Assertions.assertEquals(ArrayType.BYTE_ARRAY_TYPE, outType.getFieldType(2)); + Assertions.assertArrayEquals(new Byte[] {(byte) -3}, (Object[]) out.getField(2)); + Assertions.assertEquals(Collections.singletonMap("null", null), out.getField(3)); + Assertions.assertEquals(ArrayType.SHORT_ARRAY_TYPE, outType.getFieldType(4)); + Assertions.assertArrayEquals(new Short[] {(short) -2}, (Object[]) out.getField(4)); + Assertions.assertEquals(ArrayType.INT_ARRAY_TYPE, outType.getFieldType(5)); + Assertions.assertArrayEquals(new Integer[] {-3, 1}, (Object[]) out.getField(5)); + Assertions.assertDoesNotThrow(() -> out.getBytesSize(outType)); + } + // ==================== SQL Evaluation Tests ==================== @Test
