This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 87a1a6f299 [hive] Truncate Iceberg Hive column comments for Glue
compatibility (#8223)
87a1a6f299 is described below
commit 87a1a6f299480927214cd4adb79c6d77f83ac8ba
Author: jsingh-yelp <[email protected]>
AuthorDate: Fri Jun 12 22:39:39 2026 -0400
[hive] Truncate Iceberg Hive column comments for Glue compatibility (#8223)
This PR truncates column comments written by
IcebergHiveMetadataCommitter to 255 characters before syncing them to
Hive/Glue metadata. ErrorTrace:
```
'table.storageDescriptor.columns.29.member.comment' failed to satisfy
constraint: Member must have length less than or equal to 255 (Service:
AWSGlue; Status Code: 400; Error Code: ValidationException; Request ID:
f75bb3cb-36e2-4979-b38a-c4bec5aeb930; Proxy: null))
```
AWS Glue rejects FieldSchema.comment values longer than 255 characters,
which can cause Paimon commits to fail when Iceberg Hive metadata sync
is enabled and an upstream schema field description is too long. The
committer now preserves short/null comments unchanged and truncates
longer comments with a `...` marker while staying within the Glue limit.
---
.../iceberg/IcebergHiveMetadataCommitter.java | 15 ++++++-
.../iceberg/IcebergHiveMetadataCommitterTest.java | 49 ++++++++++++++++++++++
2 files changed, 63 insertions(+), 1 deletion(-)
diff --git
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitter.java
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitter.java
index 6d0554c8cc..60f2016070 100644
---
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitter.java
+++
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitter.java
@@ -18,6 +18,7 @@
package org.apache.paimon.iceberg;
+import org.apache.paimon.annotation.VisibleForTesting;
import org.apache.paimon.catalog.Identifier;
import org.apache.paimon.client.ClientPool;
import org.apache.paimon.fs.Path;
@@ -61,6 +62,8 @@ import static
org.apache.paimon.iceberg.IcebergCommitCallback.catalogDatabasePat
public class IcebergHiveMetadataCommitter implements IcebergMetadataCommitter {
private static final Logger LOG =
LoggerFactory.getLogger(IcebergHiveMetadataCommitter.class);
+ private static final int HIVE_COLUMN_COMMENT_MAX_LENGTH = 255;
+ private static final String TRUNCATION_MARKER = "...";
private final FileStoreTable table;
private final ClientPool<IMetaStoreClient, TException> clients;
@@ -263,6 +266,16 @@ public class IcebergHiveMetadataCommitter implements
IcebergMetadataCommitter {
return new FieldSchema(
dataField.name(),
HiveTypeUtils.toTypeInfo(dataField.type()).getTypeName(),
- dataField.description());
+ normalizeColumnComment(dataField.description()));
+ }
+
+ @VisibleForTesting
+ static String normalizeColumnComment(@Nullable String comment) {
+ if (comment == null || comment.length() <=
HIVE_COLUMN_COMMENT_MAX_LENGTH) {
+ return comment;
+ }
+
+ return comment.substring(0, HIVE_COLUMN_COMMENT_MAX_LENGTH -
TRUNCATION_MARKER.length())
+ + TRUNCATION_MARKER;
}
}
diff --git
a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitterTest.java
b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitterTest.java
new file mode 100644
index 0000000000..e7559e6ea4
--- /dev/null
+++
b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitterTest.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.iceberg;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.Arrays;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Tests for {@link IcebergHiveMetadataCommitter}. */
+class IcebergHiveMetadataCommitterTest {
+
+ @Test
+ void testNormalizeColumnComment() {
+
assertThat(IcebergHiveMetadataCommitter.normalizeColumnComment(null)).isNull();
+
+ String maxLengthComment = repeat('a', 255);
+
assertThat(IcebergHiveMetadataCommitter.normalizeColumnComment(maxLengthComment))
+ .isEqualTo(maxLengthComment);
+
+ String longComment = repeat('b', 256);
+
assertThat(IcebergHiveMetadataCommitter.normalizeColumnComment(longComment))
+ .hasSize(255)
+ .isEqualTo(repeat('b', 252) + "...");
+ }
+
+ private static String repeat(char c, int count) {
+ char[] chars = new char[count];
+ Arrays.fill(chars, c);
+ return new String(chars);
+ }
+}
