This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 0158246aba [hive] Normalize column comments written to Hive metastore
(#9073)
0158246aba is described below
commit 0158246aba05ab759ec965d37be7763bd5764604
Author: Eunbin Son <[email protected]>
AuthorDate: Thu Aug 6 23:13:08 2026 +0900
[hive] Normalize column comments written to Hive metastore (#9073)
---
.../java/org/apache/paimon/hive/HiveCatalog.java | 18 +++++-
.../org/apache/paimon/hive/HiveTableUtils.java | 38 +++++++++++-
.../iceberg/IcebergHiveMetadataCommitter.java | 16 +----
.../org/apache/paimon/hive/HiveCatalogTest.java | 68 ++++++++++++++++++++++
4 files changed, 122 insertions(+), 18 deletions(-)
diff --git
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
index 5811796e5a..7b57c6bcea 100644
---
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
+++
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
@@ -951,7 +951,7 @@ public class HiveCatalog extends AbstractCatalog {
StorageDescriptor sd = hiveTable.getSd();
List<FieldSchema> columns =
view.rowType().getFields().stream()
- .map(this::convertToFieldSchema)
+ .map(this::convertToColumnFieldSchema)
.collect(Collectors.toList());
sd.setCols(columns);
@@ -1797,7 +1797,7 @@ public class HiveCatalog extends AbstractCatalog {
List<FieldSchema> normalFields = new ArrayList<>();
for (DataField field : schema.fields()) {
if (!partitionKeys.contains(field.name())) {
- normalFields.add(convertToFieldSchema(field));
+ normalFields.add(convertToColumnFieldSchema(field));
}
}
sd.setCols(normalFields);
@@ -1819,7 +1819,7 @@ public class HiveCatalog extends AbstractCatalog {
sd.setCols(
schema.fields().stream()
- .map(this::convertToFieldSchema)
+ .map(this::convertToColumnFieldSchema)
.collect(Collectors.toList()));
}
table.setSd(sd);
@@ -1871,6 +1871,18 @@ public class HiveCatalog extends AbstractCatalog {
}
}
+ /**
+ * Converts a {@link DataField} to a Hive column, whose comment is stored
in {@code
+ * COLUMNS_V2.COMMENT} and thus has to be normalized. Use {@link
#convertToFieldSchema} for
+ * partition keys, which are stored in {@code PARTITION_KEYS.PKEY_COMMENT}
instead.
+ */
+ private FieldSchema convertToColumnFieldSchema(DataField dataField) {
+ return new FieldSchema(
+ dataField.name(),
+ HiveTypeUtils.toTypeInfo(dataField.type()).getTypeName(),
+
HiveTableUtils.normalizeColumnComment(dataField.description()));
+ }
+
private FieldSchema convertToFieldSchema(DataField dataField) {
return new FieldSchema(
dataField.name(),
diff --git
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveTableUtils.java
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveTableUtils.java
index d352b148b6..56ac5b82f4 100644
---
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveTableUtils.java
+++
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveTableUtils.java
@@ -28,6 +28,8 @@ import org.apache.hadoop.hive.metastore.api.SerDeInfo;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
+import javax.annotation.Nullable;
+
import java.util.ArrayList;
import java.util.List;
@@ -42,7 +44,41 @@ import static
org.apache.paimon.hive.HiveCatalog.HIVE_FIELD_DELIM_DEFAULT;
import static org.apache.paimon.hive.HiveCatalog.TABLE_TYPE_PROP;
import static org.apache.paimon.hive.HiveCatalog.isView;
-class HiveTableUtils {
+/** Utils for converting between Paimon and Hive tables. */
+public class HiveTableUtils {
+
+ /**
+ * Max length of a Hive column comment. The metastore stores column
comments in {@code
+ * COLUMNS_V2.COMMENT}, which is mapped to {@code VARCHAR(256)}, and AWS
Glue rejects comments
+ * longer than 255 characters.
+ */
+ private static final int HIVE_COLUMN_COMMENT_MAX_LENGTH = 255;
+
+ private static final String TRUNCATION_MARKER = "...";
+
+ /**
+ * Normalizes a column comment so that it can be stored in {@code
COLUMNS_V2.COMMENT}: line
+ * breaks are replaced by spaces and the result is truncated to {@link
+ * #HIVE_COLUMN_COMMENT_MAX_LENGTH} characters.
+ *
+ * <p>Note that this must not be applied to partition key comments, which
are stored in {@code
+ * PARTITION_KEYS.PKEY_COMMENT} with a much larger limit.
+ */
+ @Nullable
+ public static String normalizeColumnComment(@Nullable String comment) {
+ if (comment == null) {
+ return null;
+ }
+
+ String normalized = comment.replace('\n', ' ').replace('\r', ' ');
+
+ if (normalized.length() <= HIVE_COLUMN_COMMENT_MAX_LENGTH) {
+ return normalized;
+ }
+
+ return normalized.substring(0, HIVE_COLUMN_COMMENT_MAX_LENGTH -
TRUNCATION_MARKER.length())
+ + TRUNCATION_MARKER;
+ }
public static Schema tryToFormatSchema(Table hiveTable) {
if (isView(hiveTable)) {
diff --git
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitter.java
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitter.java
index aa9b3a229e..ad99382574 100644
---
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitter.java
+++
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitter.java
@@ -23,6 +23,7 @@ import org.apache.paimon.catalog.Identifier;
import org.apache.paimon.client.ClientPool;
import org.apache.paimon.fs.Path;
import org.apache.paimon.hive.HiveCatalog;
+import org.apache.paimon.hive.HiveTableUtils;
import org.apache.paimon.hive.HiveTypeUtils;
import org.apache.paimon.hive.pool.CachedClientPool;
import org.apache.paimon.iceberg.metadata.IcebergMetadata;
@@ -62,8 +63,6 @@ import static
org.apache.paimon.iceberg.IcebergCommitCallback.catalogDatabasePat
public class IcebergHiveMetadataCommitter implements IcebergMetadataCommitter {
private static final Logger LOG =
LoggerFactory.getLogger(IcebergHiveMetadataCommitter.class);
- private static final int HIVE_COLUMN_COMMENT_MAX_LENGTH = 255;
- private static final String TRUNCATION_MARKER = "...";
private final FileStoreTable table;
private final ClientPool<IMetaStoreClient, TException> clients;
@@ -271,17 +270,6 @@ public class IcebergHiveMetadataCommitter implements
IcebergMetadataCommitter {
@VisibleForTesting
static String normalizeColumnComment(@Nullable String comment) {
- if (comment == null) {
- return comment;
- }
-
- comment = comment.replace('\n', ' ').replace('\r', ' ');
-
- if (comment.length() <= HIVE_COLUMN_COMMENT_MAX_LENGTH) {
- return comment;
- }
-
- return comment.substring(0, HIVE_COLUMN_COMMENT_MAX_LENGTH -
TRUNCATION_MARKER.length())
- + TRUNCATION_MARKER;
+ return HiveTableUtils.normalizeColumnComment(comment);
}
}
diff --git
a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java
b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java
index d158f0398c..c0019880d1 100644
---
a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java
+++
b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java
@@ -48,6 +48,7 @@ import
org.apache.paimon.shade.guava30.com.google.common.collect.Lists;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.thrift.TException;
import org.junit.jupiter.api.BeforeEach;
@@ -626,6 +627,73 @@ public class HiveCatalogTest extends CatalogTestBase {
.containsEntry(partitionSpecs.get(1), false);
}
+ @Test
+ public void testCreateTableWithLongColumnComment() throws Exception {
+ String databaseName = "testCreateTableWithLongColumnComment";
+ catalog.createDatabase(databaseName, false);
+
+ String longComment = "line1\n" + repeat('a', 300);
+ String shortComment = "a short comment";
+ Identifier identifier = Identifier.create(databaseName, "table");
+ catalog.createTable(
+ identifier,
+ Schema.newBuilder()
+ .column("col", DataTypes.INT(), longComment)
+ .column("col2", DataTypes.INT(), shortComment)
+ .build(),
+ false);
+
+ // the comment mirrored to the metastore is truncated and contains no
line break
+ List<FieldSchema> cols =
+ ((HiveCatalog) catalog)
+ .getHmsClient()
+ .getTable(databaseName, "table")
+ .getSd()
+ .getCols();
+
assertThat(cols.get(0).getComment()).hasSize(255).endsWith("...").doesNotContain("\n");
+ // a short comment without line breaks is left untouched
+ assertThat(cols.get(1).getComment()).isEqualTo(shortComment);
+
+ // the Paimon schema keeps the original comment
+ assertThat(catalog.getTable(identifier).rowType().getFields())
+ .extracting(DataField::description)
+ .containsExactly(longComment, shortComment);
+ }
+
+ @Test
+ public void testCreateTableWithLongPartitionKeyComment() throws Exception {
+ String databaseName = "testCreateTableWithLongPartitionKeyComment";
+ catalog.createDatabase(databaseName, false);
+
+ // partition key comments are stored in PARTITION_KEYS.PKEY_COMMENT,
which allows longer
+ // values than COLUMNS_V2.COMMENT, so they must not be truncated
+ String longComment = repeat('a', 300);
+ Identifier identifier = Identifier.create(databaseName, "table");
+ catalog.createTable(
+ identifier,
+ Schema.newBuilder()
+ .option(METASTORE_PARTITIONED_TABLE.key(), "true")
+ .column("col", DataTypes.INT())
+ .column("dt", DataTypes.STRING(), longComment)
+ .partitionKeys("dt")
+ .build(),
+ false);
+
+ List<FieldSchema> partitionKeys =
+ ((HiveCatalog) catalog)
+ .getHmsClient()
+ .getTable(databaseName, "table")
+ .getPartitionKeys();
+ assertThat(partitionKeys).hasSize(1);
+ assertThat(partitionKeys.get(0).getComment()).isEqualTo(longComment);
+ }
+
+ private static String repeat(char c, int count) {
+ char[] chars = new char[count];
+ Arrays.fill(chars, c);
+ return new String(chars);
+ }
+
@Test
public void testCreateTableWithBlob() throws Exception {
String databaseName = "testCreateTableWithBlob";
