This is an automated email from the ASF dual-hosted git repository.
Abacn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 22c43bf2bd5 Mitigate test broken after Avro Upgrade due to AVRO-4110
(#38405)
22c43bf2bd5 is described below
commit 22c43bf2bd5d0eac8147abe9623e7f0f593c1691
Author: Yi Hu <[email protected]>
AuthorDate: Thu May 7 17:32:04 2026 -0400
Mitigate test broken after Avro Upgrade due to AVRO-4110 (#38405)
---
.../AvroReadSchemaTransformFormatProviderTest.java | 11 ++++
...WriteSchemaTransformFormatProviderTestData.java | 61 ++++++++++++++++++++++
...rquetReadSchemaTransformFormatProviderTest.java | 11 ++++
3 files changed, 83 insertions(+)
diff --git
a/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/AvroReadSchemaTransformFormatProviderTest.java
b/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/AvroReadSchemaTransformFormatProviderTest.java
index 5725ceff3a1..bfba60fb0c1 100644
---
a/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/AvroReadSchemaTransformFormatProviderTest.java
+++
b/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/AvroReadSchemaTransformFormatProviderTest.java
@@ -18,6 +18,7 @@
package org.apache.beam.sdk.io.fileschematransform;
import static
org.apache.beam.sdk.io.common.SchemaAwareJavaBeans.ALL_PRIMITIVE_DATA_TYPES_SCHEMA;
+import static
org.apache.beam.sdk.io.common.SchemaAwareJavaBeans.ARRAY_PRIMITIVE_DATA_TYPES_SCHEMA;
import static
org.apache.beam.sdk.io.fileschematransform.FileReadSchemaTransformProvider.FILEPATTERN_ROW_FIELD_NAME;
import static
org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformFormatProviderTestData.DATA;
import static
org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions.RESOLVE_FILE;
@@ -215,6 +216,16 @@ public class AvroReadSchemaTransformFormatProviderTest
readPipeline.run();
}
+ // TODO(AVRO-4110): remove this override when Beam upgraded Avro past 1.12.0
+ @Override
+ public void testArrayPrimitiveDataTypes() {
+ Schema schema = ARRAY_PRIMITIVE_DATA_TYPES_SCHEMA;
+ List<Row> rows = DATA.arrayPrimitiveDataTypesRowsAvro1120;
+ String filePath = getFilePath();
+
+ runWriteAndReadTest(schema, rows, filePath, null);
+ }
+
private static class TestDynamicDestinations
extends DynamicAvroDestinations<GenericRecord, String, GenericRecord> {
final ResourceId baseDir;
diff --git
a/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformFormatProviderTestData.java
b/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformFormatProviderTestData.java
index 4f70dca71e3..d92acaa0ac6 100644
---
a/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformFormatProviderTestData.java
+++
b/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/FileWriteSchemaTransformFormatProviderTestData.java
@@ -35,6 +35,7 @@ import static
org.apache.beam.sdk.io.common.SchemaAwareJavaBeans.timeContaining;
import static
org.apache.beam.sdk.io.common.SchemaAwareJavaBeans.timeContainingToRowFn;
import java.math.BigDecimal;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
@@ -50,6 +51,7 @@ import
org.apache.beam.sdk.io.common.SchemaAwareJavaBeans.NullableAllPrimitiveDa
import
org.apache.beam.sdk.io.common.SchemaAwareJavaBeans.SinglyNestedDataTypes;
import org.apache.beam.sdk.io.common.SchemaAwareJavaBeans.TimeContaining;
import org.apache.beam.sdk.values.Row;
+import
org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists;
import org.joda.time.Instant;
/** Shared {@link SchemaAwareJavaBeans} data to be used across various tests.
*/
@@ -60,6 +62,23 @@ class FileWriteSchemaTransformFormatProviderTestData {
/* Prevent instantiation outside this class. */
private FileWriteSchemaTransformFormatProviderTestData() {}
+ private static class ListPatcher<T> {
+ private ArrayList<T> list;
+
+ ListPatcher(List<T> list) {
+ this.list = Lists.newArrayList(list);
+ }
+
+ ArrayList<T> get() {
+ return list;
+ }
+
+ ListPatcher<T> patch(int index, T value) {
+ list.set(index, value);
+ return this;
+ }
+ }
+
final List<AllPrimitiveDataTypes> allPrimitiveDataTypesList =
Arrays.asList(
allPrimitiveDataTypes(false, BigDecimal.valueOf(1L), 1.2345,
1.2345f, 1, 1L, "a"),
@@ -188,11 +207,53 @@ class FileWriteSchemaTransformFormatProviderTestData {
Collections.emptyList(),
Collections.emptyList()));
+ // TODO(AVRO-4110): remove this workaround when Beam upgraded Avro past
1.12.0
+ final List<ArrayPrimitiveDataTypes> arrayPrimitiveDataTypesListAvro1120 =
+ new ListPatcher<>(arrayPrimitiveDataTypesList)
+ .patch(
+ 1,
+ arrayPrimitiveDataTypes(
+ Collections.emptyList(),
+ Collections.singletonList((double) Float.MAX_VALUE),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ Collections.emptyList()))
+ .patch(
+ 6,
+ arrayPrimitiveDataTypes(
+ Arrays.asList(false, true, false),
+ Arrays.asList((double) Float.MIN_VALUE, 0.0, (double)
Float.MAX_VALUE),
+ Arrays.asList(Float.MIN_VALUE, 0.0f, Float.MAX_VALUE),
+ Arrays.asList(Integer.MIN_VALUE, 0, Integer.MAX_VALUE),
+ Arrays.asList(Long.MIN_VALUE, 0L, Long.MAX_VALUE),
+ Arrays.asList(
+ Stream.generate(() ->
"🐤").limit(10).collect(Collectors.joining("")),
+ Stream.generate(() ->
"🐥").limit(10).collect(Collectors.joining("")),
+ Stream.generate(() ->
"🐣").limit(10).collect(Collectors.joining("")))))
+ .patch(
+ 7,
+ arrayPrimitiveDataTypes(
+ Stream.generate(() ->
true).limit(10).collect(Collectors.toList()),
+ Stream.generate(() -> (double) Float.MIN_VALUE)
+ .limit(10)
+ .collect(Collectors.toList()),
+ Stream.generate(() ->
Float.MIN_VALUE).limit(10).collect(Collectors.toList()),
+ Stream.generate(() ->
Integer.MIN_VALUE).limit(10).collect(Collectors.toList()),
+ Stream.generate(() ->
Long.MIN_VALUE).limit(10).collect(Collectors.toList()),
+ Stream.generate(() ->
"🐿").limit(10).collect(Collectors.toList())))
+ .get();
+
final List<Row> arrayPrimitiveDataTypesRows =
arrayPrimitiveDataTypesList.stream()
.map(arrayPrimitiveDataTypesToRowFn()::apply)
.collect(Collectors.toList());
+ final List<Row> arrayPrimitiveDataTypesRowsAvro1120 =
+ arrayPrimitiveDataTypesListAvro1120.stream()
+ .map(arrayPrimitiveDataTypesToRowFn()::apply)
+ .collect(Collectors.toList());
+
final List<SinglyNestedDataTypes> singlyNestedDataTypesNoRepeat =
allPrimitiveDataTypesList.stream()
.map(SchemaAwareJavaBeans::singlyNestedDataTypes)
diff --git
a/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/ParquetReadSchemaTransformFormatProviderTest.java
b/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/ParquetReadSchemaTransformFormatProviderTest.java
index b1d6bba06ea..bbc33698c41 100644
---
a/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/ParquetReadSchemaTransformFormatProviderTest.java
+++
b/sdks/java/io/file-schema-transform/src/test/java/org/apache/beam/sdk/io/fileschematransform/ParquetReadSchemaTransformFormatProviderTest.java
@@ -18,6 +18,7 @@
package org.apache.beam.sdk.io.fileschematransform;
import static
org.apache.beam.sdk.io.common.SchemaAwareJavaBeans.ALL_PRIMITIVE_DATA_TYPES_SCHEMA;
+import static
org.apache.beam.sdk.io.common.SchemaAwareJavaBeans.ARRAY_PRIMITIVE_DATA_TYPES_SCHEMA;
import static
org.apache.beam.sdk.io.fileschematransform.FileReadSchemaTransformProvider.FILEPATTERN_ROW_FIELD_NAME;
import static
org.apache.beam.sdk.io.fileschematransform.FileWriteSchemaTransformFormatProviderTestData.DATA;
import static org.apache.beam.sdk.transforms.Contextful.fn;
@@ -218,4 +219,14 @@ public class ParquetReadSchemaTransformFormatProviderTest
PAssert.that(output.get(FileReadSchemaTransformProvider.OUTPUT_TAG)).containsInAnyOrder(rows);
readPipeline.run();
}
+
+ // TODO(AVRO-4110): remove this override when Beam upgraded Avro past 1.12.0
+ @Override
+ public void testArrayPrimitiveDataTypes() {
+ Schema schema = ARRAY_PRIMITIVE_DATA_TYPES_SCHEMA;
+ List<Row> rows = DATA.arrayPrimitiveDataTypesRowsAvro1120;
+ String filePath = getFilePath();
+
+ runWriteAndReadTest(schema, rows, filePath, null);
+ }
}
