This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/dev/pr-12618-31100cb5301274fdb1f27ba539d2f46ade44dd9e in repository https://gitbox.apache.org/repos/asf/seatunnel.git
commit a4c291d8fd6c7b785b90334d4ec2ee68e026dd7a Author: Sepuri Sai Krishna <[email protected]> AuthorDate: Mon Oct 12 05:27:05 2026 +0000 [Fix][Transform-V2] Make calcite, nlpmodel and validator case conversion locale independent (#12618) --- .../transform/calcite/udf/BuiltinFunctions.java | 3 +- .../calcite/udf/VectorReduceFunction.java | 3 +- .../nlpmodel/ModelInvocationCacheKey.java | 3 +- .../embedding/multimodal/ModalityType.java | 7 ++- .../embedding/multimodal/PayloadFormat.java | 4 +- .../embedding/remote/doubao/DoubaoModel.java | 3 +- .../nlpmodel/llm/remote/AbstractModel.java | 3 +- .../validator/DataValidatorTransformConfig.java | 5 +- .../transform/calcite/CalciteSQLEngineTest.java | 27 +++++++++ .../embedding/DoubaoMultimodalModelTest.java | 68 ++++++++++++++++++++++ .../nlpmodel/ModelInvocationCacheKeyTest.java | 35 +++++++++++ 11 files changed, 150 insertions(+), 11 deletions(-) diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/BuiltinFunctions.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/BuiltinFunctions.java index d88eb7e72d..f878d0de6f 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/BuiltinFunctions.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/BuiltinFunctions.java @@ -27,6 +27,7 @@ import java.lang.reflect.Method; import java.lang.reflect.Modifier; import java.util.ArrayList; import java.util.List; +import java.util.Locale; import java.util.ServiceLoader; /** Discovers {@link CalciteUdf} implementations via {@link ServiceLoader} and registers them. */ @@ -51,7 +52,7 @@ public final class BuiltinFunctions { udf.getClass().getName()); return; } - String name = rawName.toUpperCase(); + String name = rawName.toUpperCase(Locale.ROOT); boolean opened = false; try { Method evalMethod = findStaticEvalMethod(udf.getClass()); diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/VectorReduceFunction.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/VectorReduceFunction.java index 6bc2655f68..1c0cbdc9af 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/VectorReduceFunction.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/calcite/udf/VectorReduceFunction.java @@ -22,6 +22,7 @@ import org.apache.seatunnel.common.utils.VectorUtils; import com.google.auto.service.AutoService; import java.nio.ByteBuffer; +import java.util.Locale; import java.util.Random; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -57,7 +58,7 @@ public class VectorReduceFunction implements CalciteUdf { } Float[] result; - switch (method.toUpperCase()) { + switch (method.toUpperCase(Locale.ROOT)) { case "TRUNCATE": result = new Float[targetDimension]; System.arraycopy(source, 0, result, 0, targetDimension); diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationCacheKey.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationCacheKey.java index d9ae64dfba..df2a4dd3d0 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationCacheKey.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationCacheKey.java @@ -21,6 +21,7 @@ import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.util.Locale; import java.util.Map; import java.util.TreeMap; @@ -141,7 +142,7 @@ public final class ModelInvocationCacheKey { if (normalized.isEmpty()) { return null; } - return normalized.toLowerCase(); + return normalized.toLowerCase(Locale.ROOT); } private static String normalizeValue(Object value) { diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/multimodal/ModalityType.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/multimodal/ModalityType.java index 0d5f337348..c40232e899 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/multimodal/ModalityType.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/multimodal/ModalityType.java @@ -23,6 +23,7 @@ import lombok.ToString; import java.util.Arrays; import java.util.List; +import java.util.Locale; /** Enumeration for multimodal modality types supported by embedding models */ @AllArgsConstructor @@ -57,7 +58,7 @@ public enum ModalityType { return null; } - String trimmedName = name.trim().toLowerCase(); + String trimmedName = name.trim().toLowerCase(Locale.ROOT); for (ModalityType type : ModalityType.values()) { if (type.name.equalsIgnoreCase(trimmedName)) { return type; @@ -75,7 +76,7 @@ public enum ModalityType { if (value == null || value.trim().isEmpty()) { return null; } - String trimmedValue = value.trim().toLowerCase(); + String trimmedValue = value.trim().toLowerCase(Locale.ROOT); String extension = ""; int lastDotIndex = trimmedValue.lastIndexOf('.'); if (lastDotIndex > 0 && lastDotIndex < trimmedValue.length() - 1) { @@ -99,7 +100,7 @@ public enum ModalityType { if (extension == null) { return false; } - return fileExtensions.contains(extension.toLowerCase()); + return fileExtensions.contains(extension.toLowerCase(Locale.ROOT)); } public enum ModalityGroup { diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/multimodal/PayloadFormat.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/multimodal/PayloadFormat.java index 60adcb3f4f..dd0ee8bd43 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/multimodal/PayloadFormat.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/multimodal/PayloadFormat.java @@ -21,6 +21,8 @@ import lombok.AllArgsConstructor; import lombok.Getter; import lombok.ToString; +import java.util.Locale; + /** Enumeration for data formats supported by multimodal embedding models */ @AllArgsConstructor @Getter @@ -37,7 +39,7 @@ public enum PayloadFormat { return URL; } for (PayloadFormat format : PayloadFormat.values()) { - if (format.name.equalsIgnoreCase(name.trim().toLowerCase())) { + if (format.name.equalsIgnoreCase(name.trim().toLowerCase(Locale.ROOT))) { return format; } } diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/remote/doubao/DoubaoModel.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/remote/doubao/DoubaoModel.java index f175b7c9b2..17ba5b502c 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/remote/doubao/DoubaoModel.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/embedding/remote/doubao/DoubaoModel.java @@ -48,6 +48,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Locale; public class DoubaoModel extends MultimodalModel { @@ -385,7 +386,7 @@ public class DoubaoModel extends MultimodalModel { fieldValue = String.format( BASE64_PARAM_TEMPLATE, - fieldSpecModalityType.getGroup().name().toLowerCase(), + fieldSpecModalityType.getGroup().name().toLowerCase(Locale.ROOT), fieldSpecModalityType.getName(), srcField.toBase64()); } diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/llm/remote/AbstractModel.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/llm/remote/AbstractModel.java index f57d66f906..877ed3f80b 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/llm/remote/AbstractModel.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/nlpmodel/llm/remote/AbstractModel.java @@ -31,6 +31,7 @@ import org.apache.seatunnel.format.json.RowToJsonConverters; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Locale; public abstract class AbstractModel implements Model { @@ -120,6 +121,6 @@ public abstract class AbstractModel implements Model { throws IOException; protected String convertData(String data) { - return outputType == SqlType.BOOLEAN ? data.toLowerCase() : data; + return outputType == SqlType.BOOLEAN ? data.toLowerCase(Locale.ROOT) : data; } } diff --git a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/validator/DataValidatorTransformConfig.java b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/validator/DataValidatorTransformConfig.java index 2bdbb742b8..d6be1ad5e9 100644 --- a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/validator/DataValidatorTransformConfig.java +++ b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/validator/DataValidatorTransformConfig.java @@ -41,6 +41,7 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; @Data @@ -165,7 +166,7 @@ public class DataValidatorTransformConfig implements Serializable { return null; } - String ruleType = String.valueOf(ruleTypeObj).toUpperCase(); + String ruleType = String.valueOf(ruleTypeObj).toUpperCase(Locale.ROOT); try { switch (ruleType) { @@ -333,7 +334,7 @@ public class DataValidatorTransformConfig implements Serializable { if (value instanceof Boolean) { return (Boolean) value; } - String stringValue = String.valueOf(value).trim().toLowerCase(); + String stringValue = String.valueOf(value).trim().toLowerCase(Locale.ROOT); return "true".equals(stringValue) || "1".equals(stringValue) || "yes".equals(stringValue); } diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/calcite/CalciteSQLEngineTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/calcite/CalciteSQLEngineTest.java index d9245aa327..efbb2e43e7 100644 --- a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/calcite/CalciteSQLEngineTest.java +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/calcite/CalciteSQLEngineTest.java @@ -40,6 +40,7 @@ import java.time.LocalDateTime; import java.time.LocalTime; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; class CalciteSQLEngineTest { @@ -4249,4 +4250,30 @@ class CalciteSQLEngineTest { Assertions.assertEquals("42", output.getOptions().get("offset")); engine.close(); } + + @Test + void testVectorReduceMethodIsLocaleIndependent() { + // VECTOR_REDUCE upper-cases the method before matching it, so a lower-case method must + // resolve. Under a Turkish default locale "random_projection" upper-cases to + // "RANDOM_PROJECTION" only when the conversion is locale independent; otherwise the i + // becomes a dotted capital I, no case matches and eval throws. + Locale original = Locale.getDefault(); + try { + Locale.setDefault(Locale.forLanguageTag("tr-TR")); + // try-with-resources rather than a trailing close(), so a failed assertion below + // still releases the engine. + try (CalciteSQLEngine engine = + createAndInit( + "SELECT VECTOR_REDUCE(vec, 2, 'random_projection') AS reduced FROM t", + "t", + singleVectorRowType())) { + Object result = + singleField(engine, new Object[] {floatVec(1.0f, 2.0f, 3.0f, 4.0f)}); + Assertions.assertNotNull(result); + Assertions.assertEquals(2, VectorUtils.toFloatArray((ByteBuffer) result).length); + } + } finally { + Locale.setDefault(original); + } + } } diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/embedding/DoubaoMultimodalModelTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/embedding/DoubaoMultimodalModelTest.java index 0489a76ace..5de86b42c4 100644 --- a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/embedding/DoubaoMultimodalModelTest.java +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/embedding/DoubaoMultimodalModelTest.java @@ -36,6 +36,7 @@ import java.util.Base64; import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.concurrent.ThreadLocalRandom; @@ -472,4 +473,71 @@ public class DoubaoMultimodalModelTest { ObjectNode inputNode = (ObjectNode) result.get("input").get(0); Assertions.assertEquals("image_url", inputNode.get("type").asText()); } + + @Test + void testBinaryBase64MimeTypeIsLocaleIndependent() { + // The base64 payload embeds the modality group as a MIME type. Lower-casing IMAGE under a + // Turkish default locale yields a dotless i, so the wire value would carry a mangled + // MIME type instead of "data:image/png". + Locale original = Locale.getDefault(); + try { + Locale.setDefault(Locale.forLanguageTag("tr-TR")); + Map<String, Object> fieldConfig = new HashMap<>(); + fieldConfig.put("field", "binary_image_field"); + fieldConfig.put("modality", "png"); + fieldConfig.put("format", "binary"); + Map.Entry<String, Object> entry = + new java.util.AbstractMap.SimpleEntry<>("binary_image_vector", fieldConfig); + + VectorFieldSpec vectorFieldSpec = new VectorFieldSpec(entry); + byte[] imageData = "mock-image-data".getBytes(java.nio.charset.StandardCharsets.UTF_8); + MultimodalFieldValue value = + new MultimodalFieldValue( + Collections.singletonList( + new SrcField( + vectorFieldSpec.getSrcFieldSpecs().get(0), imageData))); + + ObjectNode result = model.multimodalBody(value); + String url = + ((ObjectNode) result.get("input").get(0)).get("image_url").get("url").asText(); + Assertions.assertTrue( + url.startsWith("data:image/png;base64,"), + "MIME type must not depend on the default locale, but was: " + + url.substring(0, Math.min(url.length(), 24))); + Assertions.assertTrue(url.endsWith(Base64.getEncoder().encodeToString(imageData))); + } finally { + Locale.setDefault(original); + } + } + + @Test + void testFileSuffixModalityDetectionIsLocaleIndependent() { + // An upper-case suffix is lower-cased before it is matched against the extension list. + // Under a Turkish default locale the I becomes a dotless i, so the suffix matches no + // known extension and the modality is silently not detected. + Locale original = Locale.getDefault(); + try { + Locale.setDefault(Locale.forLanguageTag("tr-TR")); + Map<String, Object> fieldConfig = new HashMap<>(); + fieldConfig.put("field", "image_field"); + fieldConfig.put("format", "url"); + Map.Entry<String, Object> entry = + new java.util.AbstractMap.SimpleEntry<>("image_vector", fieldConfig); + + VectorFieldSpec vectorFieldSpec = new VectorFieldSpec(entry); + MultimodalFieldValue value = + new MultimodalFieldValue( + Collections.singletonList( + new SrcField( + vectorFieldSpec.getSrcFieldSpecs().get(0), + "https://example.com/photo.GIF";))); + + Assertions.assertEquals( + ModalityType.GIF, value.getSrcFields().get(0).getFieldSpec().getModalityType()); + Assertions.assertTrue(ModalityType.TIFF.supportsExtension("TIF")); + Assertions.assertTrue(ModalityType.AVI.supportsExtension("AVI")); + } finally { + Locale.setDefault(original); + } + } } diff --git a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationCacheKeyTest.java b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationCacheKeyTest.java index 0de16278e5..3c054dcbdc 100644 --- a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationCacheKeyTest.java +++ b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/nlpmodel/ModelInvocationCacheKeyTest.java @@ -20,6 +20,8 @@ package org.apache.seatunnel.transform.nlpmodel; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.Locale; + public class ModelInvocationCacheKeyTest { @Test @@ -141,4 +143,37 @@ public class ModelInvocationCacheKeyTest { .input("same chunk") .build(); } + + @Test + void keyIsStableAcrossSpellingsUnderAnyDefaultLocale() { + // Tokens are lower-cased so that equivalent spellings collapse to one key. Under a Turkish + // default locale the I in "OpenAI" lower-cases to a dotless i while "openai" is already + // lower case, so two equivalent configurations would produce two different keys. + Locale original = Locale.getDefault(); + try { + Locale.setDefault(Locale.forLanguageTag("tr-TR")); + String mixedCase = + ModelInvocationCacheKey.builder() + .provider("OpenAI") + .model("text-embedding-3-small") + .dimension(1536) + .modality("Image") + .format("BINARY") + .input("same chunk") + .build(); + String lowerCase = + ModelInvocationCacheKey.builder() + .provider("openai") + .model("text-embedding-3-small") + .dimension(1536) + .modality("image") + .format("binary") + .input("same chunk") + .build(); + + Assertions.assertEquals(mixedCase, lowerCase); + } finally { + Locale.setDefault(original); + } + } }
![https://example.com/photo.GIF"](https://example.com/photo.GIF"){kind=link}