This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 0eb98b005142 fix(flink): avoid quadratic map array copying during
schema evolution (#20667)
0eb98b005142 is described below
commit 0eb98b0051421125b9d1ca3ccb19072131d1e1a5
Author: Danny Chan <[email protected]>
AuthorDate: Tue Nov 17 03:05:51 2026 +0000
fix(flink): avoid quadratic map array copying during schema evolution
(#20667)
---
.../hudi/util/SchemaEvolvingRowDataProjection.java | 6 +-
.../java/org/apache/hudi/util/TypeConverters.java | 6 +-
.../apache/hudi/util/TestMapDataConversion.java | 69 ++++++++++++++++++++++
3 files changed, 77 insertions(+), 4 deletions(-)
diff --git
a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/SchemaEvolvingRowDataProjection.java
b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/SchemaEvolvingRowDataProjection.java
index ed6156274ebb..0012a482ee7e 100644
---
a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/SchemaEvolvingRowDataProjection.java
+++
b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/SchemaEvolvingRowDataProjection.java
@@ -113,9 +113,11 @@ public class SchemaEvolvingRowDataProjection implements
RowProjection {
return obj -> {
final Map<Object, Object> result = new HashMap<>();
MapData map = (MapData) obj;
+ final ArrayData keys = map.keyArray();
+ final ArrayData values = map.valueArray();
for (int i = 0; i < map.size(); i++) {
- Object keyObj = keyElementGetter.getElementOrNull(map.keyArray(), i);
- Object fromObj = valueElementGetter.getElementOrNull(map.valueArray(),
i);
+ Object keyObj = keyElementGetter.getElementOrNull(keys, i);
+ Object fromObj = valueElementGetter.getElementOrNull(values, i);
// need to handle nulls to prevent NullPointerException in
#getConversion()
Object toObj = fromObj == null ? null :
valueConverter.convert(fromObj);
result.put(keyObj, toObj);
diff --git
a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/TypeConverters.java
b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/TypeConverters.java
index eaebf74ee795..ffd1bc07c004 100644
---
a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/TypeConverters.java
+++
b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/TypeConverters.java
@@ -320,9 +320,11 @@ public class TypeConverters {
public Object convert(Object val) {
final Map<Object, Object> result = new HashMap<>();
MapData map = (MapData) val;
+ final ArrayData keys = map.keyArray();
+ final ArrayData values = map.valueArray();
for (int i = 0; i < map.size(); i++) {
- Object keyObj = keyElementGetter.getElementOrNull(map.keyArray(), i);
- Object fromObj =
valueElementGetter.getElementOrNull(map.valueArray(), i);
+ Object keyObj = keyElementGetter.getElementOrNull(keys, i);
+ Object fromObj = valueElementGetter.getElementOrNull(values, i);
// need to handle nulls to prevent NullPointerException in
#getConversion()
Object toObj = fromObj != null ? converter.convert(fromObj) : null;
result.put(keyObj, toObj);
diff --git
a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/util/TestMapDataConversion.java
b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/util/TestMapDataConversion.java
new file mode 100644
index 000000000000..85c6178a442e
--- /dev/null
+++
b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/util/TestMapDataConversion.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.util;
+
+import org.apache.flink.table.data.GenericMapData;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.MapData;
+import org.apache.flink.table.types.logical.BigIntType;
+import org.apache.flink.table.types.logical.IntType;
+import org.apache.flink.table.types.logical.MapType;
+import org.apache.flink.table.types.logical.RowType;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+
+/**
+ * Tests that map conversions materialize the input arrays only once.
+ */
+public class TestMapDataConversion {
+
+ @ParameterizedTest
+ @CsvSource({"true, 0", "true, 1", "true, 100", "false, 0", "false, 1",
"false, 100"})
+ public void testMapArraysMaterializedOnce(boolean projection, int size) {
+ Map<Integer, Integer> input = new HashMap<>();
+ Map<Integer, Long> expected = new HashMap<>();
+ for (int i = 0; i < size; i++) {
+ input.put(i, i % 2 == 0 ? i : null);
+ expected.put(i, i % 2 == 0 ? (long) i : null);
+ }
+ GenericMapData map = spy(new GenericMapData(input));
+ MapType from = new MapType(new IntType(), new IntType());
+ MapType to = new MapType(new IntType(), new BigIntType());
+ MapData result;
+ if (projection) {
+ SchemaEvolvingRowDataProjection converter =
SchemaEvolvingRowDataProjection.instance(
+ RowType.of(from), RowType.of(to), Collections.emptyMap());
+ result = converter.project(GenericRowData.of(map)).getMap(0);
+ } else {
+ result = (MapData) TypeConverters.getInstance(from, to).convert(map);
+ }
+
+ assertEquals(new GenericMapData(expected), result);
+ verify(map).keyArray();
+ verify(map).valueArray();
+ }
+}