This is an automated email from the ASF dual-hosted git repository.
davidzollo pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new e5e347591b [Fix][Transform-V2] Validate array length by element count
(#12320)
e5e347591b is described below
commit e5e347591b5e731ae0a5c3edd61b2f2f8fa198b2
Author: Goutam Adwant <[email protected]>
AuthorDate: Wed Oct 21 06:23:15 2026 -0700
[Fix][Transform-V2] Validate array length by element count (#12320)
Signed-off-by: Goutam Adwant <[email protected]>
---
.../introduction/concepts/incompatible-changes.md | 9 ++
docs/en/transforms/data-validator.md | 7 +-
.../introduction/concepts/incompatible-changes.md | 7 +
docs/zh/transforms/data-validator.md | 4 +
.../validator/rule/LengthValidationRule.java | 5 +-
.../validator/DataValidatorArrayLengthTest.java | 164 +++++++++++++++++++++
6 files changed, 193 insertions(+), 3 deletions(-)
diff --git a/docs/en/introduction/concepts/incompatible-changes.md
b/docs/en/introduction/concepts/incompatible-changes.md
index 1bc8c98f36..eff49771be 100644
--- a/docs/en/introduction/concepts/incompatible-changes.md
+++ b/docs/en/introduction/concepts/incompatible-changes.md
@@ -227,6 +227,15 @@ The Zeta SQL ARRAY function now declares TINYINT elements
as `ARRAY<TINYINT>`, m
values and previously ran successfully will now fail fast at startup with a
validation
error.
+### DataValidator Array Length
+
+The `LENGTH` rule now counts array elements rather than characters in the
+array's Java identity string. This can change which array-valued rows pass
+validation. Review array length limits that were adjusted to the previous
+behavior, and check previously skipped or routed data before replaying it.
+String, collection, binary and null handling remain unchanged. No configuration
+or state format migration is required.
+
### Zeta REST Pagination Parameter Validation
- **Behavior change: `page` and `rows` are validated on paginated endpoints**
diff --git a/docs/en/transforms/data-validator.md
b/docs/en/transforms/data-validator.md
index 4068bd4429..83991d3f9a 100644
--- a/docs/en/transforms/data-validator.md
+++ b/docs/en/transforms/data-validator.md
@@ -90,6 +90,11 @@ Parameters:
##### LENGTH
Validates the length of string, array, or collection values.
+Array length is the number of elements, including null elements. Nested arrays
+are measured by their outer length, and binary values by their number of bytes.
+String length uses Java UTF-16 code units; collection length uses its size.
+Null values pass this rule; combine it with `NOT_NULL` to reject nulls.
+
Parameters:
- `rule_type`: "LENGTH"
- `min_length` (optional): Minimum allowed length
@@ -332,4 +337,4 @@ The UDF is automatically registered using the
`@AutoService(DataValidatorUDF.cla
function_name = "EMAIL"
custom_message = "Please provide a valid email address"
}
-```
\ No newline at end of file
+```
diff --git a/docs/zh/introduction/concepts/incompatible-changes.md
b/docs/zh/introduction/concepts/incompatible-changes.md
index 45a9f1c2a0..bb0974bc39 100644
--- a/docs/zh/introduction/concepts/incompatible-changes.md
+++ b/docs/zh/introduction/concepts/incompatible-changes.md
@@ -166,6 +166,13 @@ Zeta SQL ARRAY 函数现在将 TINYINT 元素声明为 `ARRAY<TINYINT>`，与实
必须满足 min <= max。注意 `row.num = 0`（空 Source）仍然有效。此前设置了无效值且成功运行的现有作业，
将在启动时快速抛出校验错误并失败。
+### DataValidator 数组长度
+
+`LENGTH` 规则现在计算数组元素个数，而非数组的 Java 标识字符串长度。
+因此，包含数组的行可能得到与此前不同的验证结果。请检查曾为适应旧行为而调整的数组长度限制，
+并在重放数据前核对此前被跳过或路由到错误表的数据。
+字符串、集合、二进制值和空值的处理保持不变，无需迁移配置或状态格式。
+
### Zeta REST 分页参数校验
- **行为变更：分页接口开始校验 `page` 与 `rows`**
diff --git a/docs/zh/transforms/data-validator.md
b/docs/zh/transforms/data-validator.md
index 36a0e3b0ef..cdc221e0e1 100644
--- a/docs/zh/transforms/data-validator.md
+++ b/docs/zh/transforms/data-validator.md
@@ -90,6 +90,10 @@ DataValidator 转换插件会根据配置规则校验字段值，并按照指定
##### LENGTH
验证字符串、数组或集合值的长度。
+数组长度为元素个数，包含空元素。嵌套数组只计算最外层的长度，二进制值按字节数计算。
+字符串长度使用 Java UTF-16 代码单元数，集合长度使用集合大小。
+空值通过此规则；如需拒绝空值，请组合使用 `NOT_NULL`。
+
参数：
- `rule_type`: "LENGTH"
- `min_length` (可选): 最小允许长度
diff --git
a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/validator/rule/LengthValidationRule.java
b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/validator/rule/LengthValidationRule.java
index 7c9bde06c4..24d6bb5dda 100644
---
a/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/validator/rule/LengthValidationRule.java
+++
b/seatunnel-transforms-v2/src/main/java/org/apache/seatunnel/transform/validator/rule/LengthValidationRule.java
@@ -26,6 +26,7 @@ import
org.apache.seatunnel.transform.validator.ValidationResult;
import lombok.Data;
import lombok.NoArgsConstructor;
+import java.lang.reflect.Array;
import java.util.Collection;
/** Validation rule to check the length of string, array, or collection
values. */
@@ -101,8 +102,8 @@ public class LengthValidationRule implements ValidationRule
{
if (value instanceof String) {
return ((String) value).length();
}
- if (value instanceof byte[]) {
- return ((byte[]) value).length;
+ if (value.getClass().isArray()) {
+ return Array.getLength(value);
}
if (value instanceof Collection) {
return ((Collection<?>) value).size();
diff --git
a/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/validator/DataValidatorArrayLengthTest.java
b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/validator/DataValidatorArrayLengthTest.java
new file mode 100644
index 0000000000..59a54c6a89
--- /dev/null
+++
b/seatunnel-transforms-v2/src/test/java/org/apache/seatunnel/transform/validator/DataValidatorArrayLengthTest.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.transform.validator;
+
+import org.apache.seatunnel.api.configuration.ReadonlyConfig;
+import org.apache.seatunnel.api.table.catalog.CatalogTable;
+import org.apache.seatunnel.api.table.catalog.CatalogTableUtil;
+import org.apache.seatunnel.api.table.type.ArrayType;
+import org.apache.seatunnel.api.table.type.BasicType;
+import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
+import org.apache.seatunnel.api.table.type.SeaTunnelRow;
+import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
+import org.apache.seatunnel.transform.exception.TransformException;
+import org.apache.seatunnel.transform.validator.rule.LengthValidationRule;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.math.BigDecimal;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.stream.Stream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class DataValidatorArrayLengthTest {
+ static Stream<Arguments> arrays() {
+ return Stream.<Object>of(
+ new Integer[] {10, 20},
+ new Long[] {10L, 20L},
+ new String[] {"a long value", "b"},
+ new Boolean[] {true, false},
+ new BigDecimal[] {BigDecimal.ONE, BigDecimal.TEN},
+ new Object[] {null, null},
+ new Integer[][] {{1, 2, 3}, {4}},
+ new byte[] {1, 2},
+ new short[] {1, 2},
+ new int[] {1, 2},
+ new long[] {1, 2},
+ new float[] {1, 2},
+ new double[] {1, 2},
+ new boolean[] {true, false},
+ new char[] {'a', 'b'})
+ .map(value -> Arguments.of(value));
+ }
+
+ @ParameterizedTest
+ @MethodSource("arrays")
+ void countsArrayElementsInsteadOfTheirStringRepresentation(Object value) {
+ LengthValidationRule exact = new LengthValidationRule(2);
+ assertTrue(exact.validate(value, null, null).isValid());
+ assertTrue(new LengthValidationRule(2, 2).validate(value, null,
null).isValid());
+ assertFalse(new LengthValidationRule(3, null).validate(value, null,
null).isValid());
+ assertFalse(new LengthValidationRule(null, 1).validate(value, null,
null).isValid());
+ assertEquals(
+ "Expected length 1 but got 2",
+ new LengthValidationRule(1).validate(value, null,
null).getErrorMessage());
+ }
+
+ @Test
+ void preservesEmptyNullStringCollectionAndFallbackBehavior() {
+ assertTrue(
+ new LengthValidationRule(0)
+ .validate(new Integer[0], ArrayType.INT_ARRAY_TYPE,
null)
+ .isValid());
+ assertTrue(new LengthValidationRule(0).validate(new byte[0], null,
null).isValid());
+ assertTrue(
+ new LengthValidationRule(2)
+ .validate(null, ArrayType.INT_ARRAY_TYPE, null)
+ .isValid());
+ assertTrue(
+ new LengthValidationRule(2).validate("ab",
BasicType.STRING_TYPE, null).isValid());
+ assertTrue(
+ new LengthValidationRule(2)
+ .validate("\uD83D\uDE00", BasicType.STRING_TYPE, null)
+ .isValid());
+ assertTrue(
+ new LengthValidationRule(2)
+ .validate(Arrays.asList("a", "b"), null, null)
+ .isValid());
+ assertTrue(new LengthValidationRule(2).validate(12,
BasicType.INT_TYPE, null).isValid());
+ }
+
+ @Test
+ void preservesCustomErrorMessage() {
+ LengthValidationRule rule = new LengthValidationRule(1);
+ rule.setCustomMessage("wrong item count");
+ assertEquals(
+ "wrong item count",
+ rule.validate(new Integer[] {1, 2}, ArrayType.INT_ARRAY_TYPE,
null)
+ .getErrorMessage());
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"FAIL", "SKIP", "ROUTE_TO_TABLE"})
+ void keepsValidArrayRowsInTheNormalPipeline(String policy) {
+ DataValidatorTransform transform = transform(policy);
+ SeaTunnelRow row = new SeaTunnelRow(new Object[] {new Integer[] {10,
20}});
+ row.setTableId("db.source");
+ SeaTunnelRow result = transform.map(row);
+ assertSame(row, result);
+ assertEquals("db.source", result.getTableId());
+ assertEquals(
+ ArrayType.INT_ARRAY_TYPE,
+
transform.getProducedCatalogTable().getSeaTunnelRowType().getFieldType(0));
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"FAIL", "SKIP", "ROUTE_TO_TABLE"})
+ void preservesInvalidRowPolicies(String policy) {
+ DataValidatorTransform transform = transform(policy);
+ SeaTunnelRow row = new SeaTunnelRow(new Object[] {new Integer[] {10}});
+ if ("FAIL".equals(policy)) {
+ assertThrows(TransformException.class, () -> transform.map(row));
+ } else if ("SKIP".equals(policy)) {
+ assertNull(transform.map(row));
+ } else {
+ assertEquals("db.invalid", transform.map(row).getTableId());
+ }
+ }
+
+ private static DataValidatorTransform transform(String policy) {
+ Map<String, Object> rule = new HashMap<>();
+ rule.put("field_name", "items");
+ rule.put("rule_type", "LENGTH");
+ rule.put("exact_length", 2);
+ Map<String, Object> options = new HashMap<>();
+ options.put("field_rules", Collections.singletonList(rule));
+ options.put("row_error_handle_way", policy);
+ options.put("row_error_handle_way.error_table", "invalid");
+ SeaTunnelRowType rowType =
+ new SeaTunnelRowType(
+ new String[] {"items"},
+ new SeaTunnelDataType<?>[] {ArrayType.INT_ARRAY_TYPE});
+ CatalogTable table =
+ CatalogTableUtil.getCatalogTable("catalog", "db", null,
"source", rowType);
+ return new DataValidatorTransform(ReadonlyConfig.fromMap(options),
table);
+ }
+}
