This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/dev/pr-12495-80f4ec15dc96d32040c032833db75e21249c00ba in repository https://gitbox.apache.org/repos/asf/seatunnel.git
commit a89de9b70419116b47be4ecb4e018f12a942bfc6 Author: Sepuri Sai Krishna <[email protected]> AuthorDate: Mon Oct 26 14:24:43 2026 +0000 [Fix][Transform-V2] Make SQL transform keyword lookup locale independent (#12495) --- .../seatunnel/transform/sql/SQLTransform.java | 3 +- .../transform/sql/zeta/ZetaSQLFunction.java | 7 ++- .../seatunnel/transform/sql/zeta/ZetaSQLType.java | 5 +- .../transform/sql/zeta/functions/CastFunction.java | 3 +- .../sql/zeta/functions/DateTimeFunction.java | 8 +-- .../sql/zeta/functions/SystemFunction.java | 4 +- .../sql/zeta/functions/VectorFunction.java | 3 +- .../seatunnel/transform/sql/SQLTransformTest.java | 28 +++++++++ .../transform/sql/zeta/ZetaSQLEngineTest.java | 73 ++++++++++++++++++++++ .../sql/zeta/functions/DateTimeFunctionsTest.java | 36 +++++++++++ .../sql/zeta/functions/VectorFunctionTest.java | 18 ++++++ 11 files changed, 175 insertions(+), 13 deletions(-) diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/SQLTransform.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/SQLTransform.java index 2014c1e07a..a87e50e7d8 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/SQLTransform.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/SQLTransform.java @@ -49,6 +49,7 @@ import lombok.extern.slf4j.Slf4j; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Locale; import java.util.stream.Collectors; import static org.apache.seatunnel.transform.sql.SQLEngineFactory.EngineType.ZETA; @@ -81,7 +82,7 @@ public class SQLTransform extends AbstractCatalogSupportFlatMapTransform super(catalogTable); this.query = config.get(KEY_QUERY); if (config.getOptional(KEY_ENGINE).isPresent()) { - this.engineType = EngineType.valueOf(config.get(KEY_ENGINE).toUpperCase()); + this.engineType = EngineType.valueOf(config.get(KEY_ENGINE).toUpperCase(Locale.ROOT)); } else { this.engineType = ZETA; } diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLFunction.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLFunction.java index 5a69ffd86b..2aa8270821 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLFunction.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLFunction.java @@ -78,6 +78,7 @@ import java.time.temporal.TemporalAccessor; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Locale; import java.util.Map; import static java.util.UUID.randomUUID; @@ -481,7 +482,7 @@ public class ZetaSQLFunction { public Object executeFunctionExpr( String functionName, List<Object> args, Expression expression) { SeaTunnelDataType<?> targetType = zetaSQLType.getExpressionType(expression); - switch (functionName.toUpperCase()) { + switch (functionName.toUpperCase(Locale.ROOT)) { case ASCII: return StringFunction.ascii(args); case BIT_LENGTH: @@ -717,7 +718,7 @@ public class ZetaSQLFunction { } public Object executeTimeKeyExpr(String timeKeyExpr) { - switch (timeKeyExpr.toUpperCase()) { + switch (timeKeyExpr.toUpperCase(Locale.ROOT)) { case CURRENT_DATE: case CURRENT_DATE_P: return DateTimeFunction.currentDate(); @@ -737,7 +738,7 @@ public class ZetaSQLFunction { String dataType = castExpression.getColDataType().getDataType(); List<Object> args = new ArrayList<>(2); args.add(arg); - args.add(dataType.toUpperCase()); + args.add(dataType.toUpperCase(Locale.ROOT)); if (dataType.equalsIgnoreCase("DECIMAL")) { List<String> ps = castExpression.getColDataType().getArgumentsStringList(); args.add(Integer.parseInt(ps.get(0))); diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLType.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLType.java index 4d7adedf5a..5c6a4a7189 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLType.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLType.java @@ -66,6 +66,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; import java.util.List; +import java.util.Locale; import java.util.stream.Collectors; public class ZetaSQLType { @@ -320,7 +321,7 @@ public class ZetaSQLType { } private SeaTunnelDataType<?> getFunctionType(Function function) { - switch (function.getName().toUpperCase()) { + switch (function.getName().toUpperCase(Locale.ROOT)) { case ZetaSQLFunction.CHAR: case ZetaSQLFunction.CHR: case ZetaSQLFunction.CONCAT: @@ -571,7 +572,7 @@ public class ZetaSQLType { } private SeaTunnelDataType<?> getTimeKeyExprType(TimeKeyExpression timeKeyExpression) { - switch (timeKeyExpression.getStringValue().toUpperCase()) { + switch (timeKeyExpression.getStringValue().toUpperCase(Locale.ROOT)) { case ZetaSQLFunction.CURRENT_DATE: case ZetaSQLFunction.CURRENT_DATE_P: return LocalTimeType.LOCAL_DATE_TYPE; diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/CastFunction.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/CastFunction.java index 03335f6d45..de5b6b2d4b 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/CastFunction.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/CastFunction.java @@ -30,6 +30,7 @@ import net.sf.jsqlparser.statement.create.table.ColDataType; import java.util.Arrays; import java.util.List; +import java.util.Locale; public class CastFunction { @@ -90,7 +91,7 @@ public class CastFunction { public static SeaTunnelDataType<?> getCastType(SqlType originType, ColDataType colDataType) { String dataType = colDataType.getDataType(); - switch (dataType.toUpperCase()) { + switch (dataType.toUpperCase(Locale.ROOT)) { case DECIMAL: List<String> ps = colDataType.getArgumentsStringList(); return new DecimalType(Integer.parseInt(ps.get(0)), Integer.parseInt(ps.get(1))); diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/DateTimeFunction.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/DateTimeFunction.java index b179da6239..70b96dc594 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/DateTimeFunction.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/DateTimeFunction.java @@ -67,7 +67,7 @@ public class DateTimeFunction { if (args.size() >= 3) { String df = (String) args.get(2); if (df != null) { - datetimeField = df.toUpperCase(); + datetimeField = df.toUpperCase(Locale.ROOT); } } switch (datetimeField) { @@ -182,7 +182,7 @@ public class DateTimeFunction { if (args.size() >= 3) { String df = (String) args.get(2); if (df != null) { - datetimeField = df.toUpperCase(); + datetimeField = df.toUpperCase(Locale.ROOT); } } @@ -261,7 +261,7 @@ public class DateTimeFunction { if (args.size() >= 2) { String df = (String) args.get(1); if (df != null) { - datetimeField = df.toUpperCase(); + datetimeField = df.toUpperCase(Locale.ROOT); } } int year = datetime.getYear(); @@ -381,7 +381,7 @@ public class DateTimeFunction { return null; } String datetimeField = (String) args.get(1); - switch (datetimeField.toUpperCase()) { + switch (datetimeField.toUpperCase(Locale.ROOT)) { case "YEAR": if (datetime instanceof LocalDate) { return ((LocalDate) datetime).getYear(); diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/SystemFunction.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/SystemFunction.java index 787d91ba70..0219e03bf4 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/SystemFunction.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/SystemFunction.java @@ -40,6 +40,7 @@ import java.time.format.DateTimeParseException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Locale; public class SystemFunction { /** @@ -346,7 +347,8 @@ public class SystemFunction { String.format("Unsupported CAST AS Boolean: %s", v1)); } } else if (v1 instanceof String) { - if (Arrays.asList("TRUE", "FALSE").contains(v1.toString().toUpperCase())) { + if (Arrays.asList("TRUE", "FALSE") + .contains(v1.toString().toUpperCase(Locale.ROOT))) { return Boolean.parseBoolean(v1.toString()); } else { throw new TransformException( diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/VectorFunction.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/VectorFunction.java index 1980d6f668..d75dc21534 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/VectorFunction.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/sql/zeta/functions/VectorFunction.java @@ -24,6 +24,7 @@ import org.apache.seatunnel.transform.exception.TransformException; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Random; import java.util.stream.IntStream; @@ -269,7 +270,7 @@ public class VectorFunction { return null; } - switch (method.toUpperCase()) { + switch (method.toUpperCase(Locale.ROOT)) { case "TRUNCATE": return vectorTruncate(vectorData, targetDimension); case "RANDOM_PROJECTION": diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/SQLTransformTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/SQLTransformTest.java index 96ea6ae8f0..d68072bda8 100644 --- a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/SQLTransformTest.java +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/SQLTransformTest.java @@ -21,6 +21,7 @@ import org.apache.seatunnel.api.common.error.RowErrorClassification; import org.apache.seatunnel.api.configuration.ReadonlyConfig; import org.apache.seatunnel.api.table.catalog.CatalogTable; import org.apache.seatunnel.api.table.catalog.CatalogTableUtil; +import org.apache.seatunnel.api.table.catalog.Column; import org.apache.seatunnel.api.table.catalog.PhysicalColumn; import org.apache.seatunnel.api.table.catalog.TableIdentifier; import org.apache.seatunnel.api.table.catalog.TableSchema; @@ -39,10 +40,13 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Objects; +import java.util.stream.Collectors; public class SQLTransformTest { @@ -1235,4 +1239,28 @@ public class SQLTransformTest { result = sqlTransform.transformRow(new SeaTunnelRow(new Object[] {-1})); Assertions.assertNull(result); } + + @Test + public void testEngineOptionValueIsLocaleIndependent() { + Locale original = Locale.getDefault(); + try { + // In tr-TR "internal".toUpperCase() is "\u0130NTERNAL", which EngineType.valueOf + // rejects. + Locale.setDefault(Locale.forLanguageTag("tr-TR")); + + HashMap<String, Object> options = new HashMap<>(); + options.put("query", "select id, name, age from dual"); + options.put("engine", "internal"); + + SQLTransform sqlTransform = + new SQLTransform(ReadonlyConfig.fromMap(options), getCatalogTable()); + Assertions.assertEquals( + Arrays.asList("id", "name", "age"), + sqlTransform.transformTableSchema().getColumns().stream() + .map(Column::getName) + .collect(Collectors.toList())); + } finally { + Locale.setDefault(original); + } + } } diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLEngineTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLEngineTest.java index d508d2d1b6..5252d1363a 100644 --- a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLEngineTest.java +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/zeta/ZetaSQLEngineTest.java @@ -32,6 +32,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Locale; public class ZetaSQLEngineTest { @@ -605,4 +606,76 @@ public class ZetaSQLEngineTest { sql + " should widen to BIGINT and keep the value"); } } + + @Test + public void testUpperCaseSqlIsUnaffectedByTheDefaultLocale() { + Locale original = Locale.getDefault(); + try { + // The counterpart of the test below, and the control behind this PR's claim that an + // all-uppercase spelling is safe on every locale: Turkish uppercasing only moves + // lowercase "i" to "İ", so a keyword that is already uppercase is unchanged. + // Pinned because the natural way to regress these sites is to compare + // toLowerCase() against lowercase constants, which keeps lowercase SQL working while + // breaking uppercase SQL, so the test below would stay green on its own. + Locale.setDefault(Locale.forLanguageTag("tr-TR")); + + SeaTunnelRowType rowType = simpleRowType(); + ZetaSQLEngine engine = new ZetaSQLEngine(); + engine.init( + "test", + "test", + rowType, + "SELECT SIGN(age) AS s, CAST(age AS INT) AS c," + + " CURRENT_TIMESTAMP AS ts FROM test"); + + SeaTunnelRowType outType = engine.typeMapping(new ArrayList<>()); + Assertions.assertArrayEquals(new String[] {"s", "c", "ts"}, outType.getFieldNames()); + + List<SeaTunnelRow> outRows = + engine.transformBySQL(new SeaTunnelRow(new Object[] {1, "Alice", 20}), outType); + Assertions.assertNotNull(outRows); + Assertions.assertEquals(1, outRows.size()); + + SeaTunnelRow outRow = outRows.get(0); + Assertions.assertEquals(1, ((Number) outRow.getField(0)).intValue()); + Assertions.assertEquals(20, outRow.getField(1)); + Assertions.assertNotNull(outRow.getField(2)); + } finally { + Locale.setDefault(original); + } + } + + @Test + public void testLowerCaseSqlResolvesUnderAnyDefaultLocale() { + Locale original = Locale.getDefault(); + try { + // In tr-TR "sin".toUpperCase() is "S\u0130N" and "int".toUpperCase() is "\u0130NT", + // so a default-locale conversion no longer matches the dispatch keywords. + Locale.setDefault(Locale.forLanguageTag("tr-TR")); + + SeaTunnelRowType rowType = simpleRowType(); + ZetaSQLEngine engine = new ZetaSQLEngine(); + engine.init( + "test", + "test", + rowType, + "select sign(age) as s, cast(age as int) as c," + + " current_timestamp as ts from test"); + + SeaTunnelRowType outType = engine.typeMapping(new ArrayList<>()); + Assertions.assertArrayEquals(new String[] {"s", "c", "ts"}, outType.getFieldNames()); + + List<SeaTunnelRow> outRows = + engine.transformBySQL(new SeaTunnelRow(new Object[] {1, "Alice", 20}), outType); + Assertions.assertNotNull(outRows); + Assertions.assertEquals(1, outRows.size()); + + SeaTunnelRow outRow = outRows.get(0); + Assertions.assertEquals(1, ((Number) outRow.getField(0)).intValue()); + Assertions.assertEquals(20, outRow.getField(1)); + Assertions.assertNotNull(outRow.getField(2)); + } finally { + Locale.setDefault(original); + } + } } diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/zeta/functions/DateTimeFunctionsTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/zeta/functions/DateTimeFunctionsTest.java index 3b67ebe871..448fe1e78d 100644 --- a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/zeta/functions/DateTimeFunctionsTest.java +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/zeta/functions/DateTimeFunctionsTest.java @@ -36,6 +36,7 @@ import java.time.LocalDateTime; import java.time.ZoneId; import java.util.Collections; import java.util.List; +import java.util.Locale; public class DateTimeFunctionsTest { @@ -490,4 +491,39 @@ public class DateTimeFunctionsTest { LocalDateTime.now()); Assertions.assertEquals(LocalDate.of(2024, 6, 15), row3.getField(0)); } + + @Test + public void testDatetimeFieldIsLocaleIndependent() { + Locale original = Locale.getDefault(); + try { + // In tr-TR "minute".toUpperCase() is "M\u0130NUTE", which matches no datetime field. + Locale.setDefault(Locale.forLanguageTag("tr-TR")); + + SeaTunnelRowType rowType = + new SeaTunnelRowType( + new String[] {"dt1", "dt2"}, + new SeaTunnelDataType[] { + LocalTimeType.LOCAL_DATE_TIME_TYPE, + LocalTimeType.LOCAL_DATE_TIME_TYPE + }); + + SeaTunnelRow outRow = + runSql( + "select DATEADD(dt1, 1, 'minute') as added," + + " DATEDIFF(dt2, dt1, 'minute') as diff," + + " DATE_TRUNC(dt1, 'minute') as truncated," + + " EXTRACT(minute FROM dt1) as extracted" + + " from dual", + rowType, + LocalDateTime.of(2024, 1, 15, 10, 30, 45), + LocalDateTime.of(2024, 1, 15, 10, 25, 45)); + + Assertions.assertEquals(LocalDateTime.of(2024, 1, 15, 10, 31, 45), outRow.getField(0)); + Assertions.assertEquals(5L, outRow.getField(1)); + Assertions.assertEquals(LocalDateTime.of(2024, 1, 15, 10, 30, 0), outRow.getField(2)); + Assertions.assertEquals(30, outRow.getField(3)); + } finally { + Locale.setDefault(original); + } + } } diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/zeta/functions/VectorFunctionTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/zeta/functions/VectorFunctionTest.java index 025da4df42..a5d3fc705e 100644 --- a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/zeta/functions/VectorFunctionTest.java +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/sql/zeta/functions/VectorFunctionTest.java @@ -38,6 +38,7 @@ import java.nio.ByteOrder; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.Locale; import java.util.stream.Stream; public class VectorFunctionTest { @@ -376,4 +377,21 @@ public class VectorFunctionTest { Assertions.assertNull(VectorFunction.vectorNormalize(null)); } + + @Test + public void testVectorReduceMethodIsLocaleIndependent() { + Locale original = Locale.getDefault(); + try { + // In tr-TR "random_projection".toUpperCase() is "RANDOM_PROJECT\u0130ON". + Locale.setDefault(Locale.forLanguageTag("tr-TR")); + + Float[] source = new Float[] {1.0f, 2.0f, 3.0f, 4.0f}; + Object reduced = + VectorFunction.vectorReduce( + VectorUtils.toByteBuffer(source), 2, "random_projection"); + Assertions.assertEquals(2, VectorUtils.toFloatArray((ByteBuffer) reduced).length); + } finally { + Locale.setDefault(original); + } + } }
