This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 8d45fc4953 [core] Fix crash when debug logging rows with complex
fields (#10807)
8d45fc4953 is described below
commit 8d45fc49539f18d02ef49fb779ff68d4a741cf7b
Author: Jiao Mingye <[email protected]>
AuthorDate: Mon Oct 26 14:26:21 2026 +0800
[core] Fix crash when debug logging rows with complex fields (#10807)
---
.../apache/paimon/data/columnar/ColumnarRow.java | 5 +
.../paimon/data/columnar/ColumnarRowTest.java | 43 +++++++
.../src/main/java/org/apache/paimon/KeyValue.java | 95 ++++++++++++--
.../apache/paimon/io/KeyValueDataFileWriter.java | 2 +-
.../test/java/org/apache/paimon/KeyValueTest.java | 141 +++++++++++++++++++++
5 files changed, 275 insertions(+), 11 deletions(-)
diff --git
a/paimon-common/src/main/java/org/apache/paimon/data/columnar/ColumnarRow.java
b/paimon-common/src/main/java/org/apache/paimon/data/columnar/ColumnarRow.java
index 3dc4e5f93a..9fb9e734b6 100644
---
a/paimon-common/src/main/java/org/apache/paimon/data/columnar/ColumnarRow.java
+++
b/paimon-common/src/main/java/org/apache/paimon/data/columnar/ColumnarRow.java
@@ -259,6 +259,11 @@ public final class ColumnarRow implements InternalRow,
DataSetters, Serializable
"ColumnarRowData do not support hashCode, please hash fields
one by one!");
}
+ @Override
+ public String toString() {
+ return getClass().getName() + "@" +
Integer.toHexString(super.hashCode());
+ }
+
public ColumnarRow copy(ColumnVector[] vectors) {
VectorizedColumnBatch vectorizedColumnBatchCopy =
vectorizedColumnBatch.copy(vectors);
ColumnarRow columnarRow = new ColumnarRow(vectorizedColumnBatchCopy,
rowId);
diff --git
a/paimon-common/src/test/java/org/apache/paimon/data/columnar/ColumnarRowTest.java
b/paimon-common/src/test/java/org/apache/paimon/data/columnar/ColumnarRowTest.java
new file mode 100644
index 0000000000..a52dff0916
--- /dev/null
+++
b/paimon-common/src/test/java/org/apache/paimon/data/columnar/ColumnarRowTest.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.data.columnar;
+
+import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.data.columnar.heap.HeapIntVector;
+import org.apache.paimon.data.columnar.heap.HeapRowVector;
+
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatCode;
+
+/** Tests for {@link ColumnarRow}. */
+public class ColumnarRowTest {
+
+ @Test
+ public void testToStringDoesNotUseHashCode() {
+ HeapIntVector intVector = new HeapIntVector(1);
+ intVector.setInt(0, 42);
+ InternalRow row = new HeapRowVector(1, intVector).getRow(0);
+
+ // Object#toString delegates to hashCode, which is unsupported for
columnar rows
+ assertThatCode(row::toString).doesNotThrowAnyException();
+ assertThat(row.toString()).startsWith(ColumnarRow.class.getName());
+ }
+}
diff --git a/paimon-core/src/main/java/org/apache/paimon/KeyValue.java
b/paimon-core/src/main/java/org/apache/paimon/KeyValue.java
index 4c4ff61c24..dc0d3355ea 100644
--- a/paimon-core/src/main/java/org/apache/paimon/KeyValue.java
+++ b/paimon-core/src/main/java/org/apache/paimon/KeyValue.java
@@ -19,17 +19,25 @@
package org.apache.paimon;
import org.apache.paimon.annotation.VisibleForTesting;
+import org.apache.paimon.data.InternalArray;
+import org.apache.paimon.data.InternalMap;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.data.serializer.InternalRowSerializer;
+import org.apache.paimon.types.ArrayType;
import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.DataType;
+import org.apache.paimon.types.IntType;
+import org.apache.paimon.types.MapType;
+import org.apache.paimon.types.MultisetType;
import org.apache.paimon.types.RowKind;
import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.InternalRowUtils;
+import javax.annotation.Nullable;
+
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
import static org.apache.paimon.table.SpecialFields.LEVEL;
import static org.apache.paimon.table.SpecialFields.SEQUENCE_NUMBER;
@@ -191,13 +199,80 @@ public class KeyValue {
}
public static String rowDataToString(InternalRow row, RowType type) {
- return IntStream.range(0, type.getFieldCount())
- .mapToObj(
- i ->
- String.valueOf(
-
InternalRowUtils.createNullCheckingFieldGetter(
- type.getTypeAt(i), i)
- .getFieldOrNull(row)))
- .collect(Collectors.joining(", "));
+ StringBuilder builder = new StringBuilder();
+ for (int i = 0; i < type.getFieldCount(); i++) {
+ if (i > 0) {
+ builder.append(", ");
+ }
+ DataType fieldType = type.getTypeAt(i);
+ builder.append(
+ valueToString(
+
InternalRowUtils.createNullCheckingFieldGetter(fieldType, i)
+ .getFieldOrNull(row),
+ fieldType));
+ }
+ return builder.toString();
+ }
+
+ /**
+ * Converts a value of the given type to a string. Complex values are
converted field by field
+ * instead of with {@link Object#toString()}, which is not supported by
internal data structures
+ * like {@code ColumnarRow}.
+ */
+ private static String valueToString(@Nullable Object value, DataType type)
{
+ if (value == null) {
+ return "null";
+ }
+ switch (type.getTypeRoot()) {
+ case ARRAY:
+ return arrayToString((InternalArray) value, ((ArrayType)
type).getElementType());
+ case MULTISET:
+ // a multiset is internally a map from element to INT
multiplicity
+ return mapToString(
+ (InternalMap) value,
+ ((MultisetType) type).getElementType(),
+ new IntType(false));
+ case MAP:
+ return mapToString(
+ (InternalMap) value,
+ ((MapType) type).getKeyType(),
+ ((MapType) type).getValueType());
+ case ROW:
+ return "(" + rowDataToString((InternalRow) value, (RowType)
type) + ")";
+ case BINARY:
+ case VARBINARY:
+ return Arrays.toString((byte[]) value);
+ default:
+ return String.valueOf(value);
+ }
+ }
+
+ private static String arrayToString(InternalArray array, DataType
elementType) {
+ InternalArray.ElementGetter elementGetter =
InternalArray.createElementGetter(elementType);
+ StringBuilder builder = new StringBuilder("[");
+ for (int i = 0; i < array.size(); i++) {
+ if (i > 0) {
+ builder.append(", ");
+ }
+ builder.append(valueToString(elementGetter.getElementOrNull(array,
i), elementType));
+ }
+ return builder.append(']').toString();
+ }
+
+ private static String mapToString(InternalMap map, DataType keyType,
DataType valueType) {
+ InternalArray keys = map.keyArray();
+ InternalArray values = map.valueArray();
+ InternalArray.ElementGetter keyGetter =
InternalArray.createElementGetter(keyType);
+ InternalArray.ElementGetter valueGetter =
InternalArray.createElementGetter(valueType);
+ StringBuilder builder = new StringBuilder("{");
+ for (int i = 0; i < keys.size(); i++) {
+ if (i > 0) {
+ builder.append(", ");
+ }
+ builder.append(valueToString(keyGetter.getElementOrNull(keys, i),
keyType));
+ builder.append('=');
+ builder.append(valueToString(valueGetter.getElementOrNull(values,
i), valueType));
+ }
+ return builder.append('}').toString();
}
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/io/KeyValueDataFileWriter.java
b/paimon-core/src/main/java/org/apache/paimon/io/KeyValueDataFileWriter.java
index 34b6ee34f1..133a7af272 100644
--- a/paimon-core/src/main/java/org/apache/paimon/io/KeyValueDataFileWriter.java
+++ b/paimon-core/src/main/java/org/apache/paimon/io/KeyValueDataFileWriter.java
@@ -141,7 +141,7 @@ public abstract class KeyValueDataFileWriter
}
if (LOG.isDebugEnabled()) {
- LOG.debug("Write to Path " + path + " key value " +
kv.toString(keyType, valueType));
+ LOG.debug("Write to Path {} key value {}", path,
kv.toString(keyType, valueType));
}
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/KeyValueTest.java
b/paimon-core/src/test/java/org/apache/paimon/KeyValueTest.java
new file mode 100644
index 0000000000..f29d1bd179
--- /dev/null
+++ b/paimon-core/src/test/java/org/apache/paimon/KeyValueTest.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon;
+
+import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.data.GenericArray;
+import org.apache.paimon.data.GenericMap;
+import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.data.columnar.ColumnarRow;
+import org.apache.paimon.data.columnar.heap.HeapIntVector;
+import org.apache.paimon.data.columnar.heap.HeapRowVector;
+import org.apache.paimon.data.serializer.InternalRowSerializer;
+import org.apache.paimon.types.ArrayType;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.IntType;
+import org.apache.paimon.types.MapType;
+import org.apache.paimon.types.MultisetType;
+import org.apache.paimon.types.RowKind;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.types.VarBinaryType;
+import org.apache.paimon.types.VarCharType;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.Collections;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Tests for {@link KeyValue}. */
+public class KeyValueTest {
+
+ private static final String COMPLEX_ROW_STRING =
+ "[1, 2, null], {1=2}, (7), [(3)], {k=[1, 2]}, {5=2}, [1, 2], null";
+
+ @Test
+ public void testRowDataToStringScalars() {
+ RowType rowType =
+ RowType.of(
+ new DataField(0, "i", new IntType()),
+ new DataField(1, "s", new
VarCharType(VarCharType.MAX_LENGTH)),
+ new DataField(2, "n", new IntType()));
+ assertThat(
+ KeyValue.rowDataToString(
+ GenericRow.of(1, BinaryString.fromString("a"),
null), rowType))
+ .isEqualTo("1, a, null");
+ }
+
+ @Test
+ public void testRowDataToStringComplexTypes() {
+ RowType rowType = complexRowType();
+ GenericRow row = complexRow();
+
+ assertThat(KeyValue.rowDataToString(row,
rowType)).isEqualTo(COMPLEX_ROW_STRING);
+
+ // binary rows format identically
+ InternalRow binaryRow = new
InternalRowSerializer(rowType).toBinaryRow(row).copy();
+ assertThat(KeyValue.rowDataToString(binaryRow,
rowType)).isEqualTo(COMPLEX_ROW_STRING);
+ }
+
+ @Test
+ public void testRowDataToStringNestedColumnarRow() {
+ HeapIntVector intVector = new HeapIntVector(1);
+ intVector.setInt(0, 7);
+ InternalRow columnarRow = new HeapRowVector(1, intVector).getRow(0);
+ assertThat(columnarRow).isInstanceOf(ColumnarRow.class);
+
+ RowType rowType =
+ RowType.of(new DataField(0, "r", RowType.of(new DataField(0,
"x", new IntType()))));
+
+ // nested rows read from columnar formats are ColumnarRows, which must
not be formatted
+ // with Object#toString
+ assertThat(KeyValue.rowDataToString(GenericRow.of(columnarRow),
rowType)).isEqualTo("(7)");
+ }
+
+ @Test
+ public void testKeyValueToString() {
+ RowType keyType = RowType.of(new DataField(0, "k", new IntType()));
+ RowType valueType = RowType.of(new DataField(0, "v", new
ArrayType(true, new IntType())));
+ KeyValue kv =
+ new KeyValue()
+ .replace(
+ GenericRow.of(1),
+ 5L,
+ RowKind.INSERT,
+ GenericRow.of(new GenericArray(new Object[]
{1, 2})))
+ .setLevel(2);
+
+ assertThat(kv.toString(keyType, valueType))
+ .isEqualTo("{kind: INSERT, seq: 5, key: (1), value: ([1, 2]),
level: 2}");
+ }
+
+ private static RowType complexRowType() {
+ RowType nestedRowType = RowType.of(new DataField(0, "z", new
IntType()));
+ return RowType.of(
+ new DataField(0, "arr", new ArrayType(true, new IntType())),
+ new DataField(1, "map", new MapType(new IntType(), new
IntType())),
+ new DataField(2, "row", nestedRowType),
+ new DataField(3, "arrRow", new ArrayType(true, nestedRowType)),
+ new DataField(
+ 4,
+ "mapArr",
+ new MapType(
+ new VarCharType(VarCharType.MAX_LENGTH),
+ new ArrayType(true, new IntType()))),
+ new DataField(5, "ms", new MultisetType(new IntType())),
+ new DataField(6, "bin", new
VarBinaryType(VarBinaryType.MAX_LENGTH)),
+ new DataField(7, "nullMap", new MapType(new IntType(), new
IntType())));
+ }
+
+ private static GenericRow complexRow() {
+ return GenericRow.of(
+ new GenericArray(new Object[] {1, 2, null}),
+ new GenericMap(Collections.singletonMap(1, 2)),
+ GenericRow.of(7),
+ new GenericArray(new Object[] {GenericRow.of(3)}),
+ new GenericMap(
+ Collections.singletonMap(
+ BinaryString.fromString("k"),
+ new GenericArray(new Object[] {1, 2}))),
+ new GenericMap(Collections.singletonMap(5, 2)),
+ new byte[] {1, 2},
+ null);
+ }
+}
