parisni opened a new issue, #5489:
URL: https://github.com/apache/hudi/issues/5489
hudi 0.11.0
spark 3.2.1 / spark 2.4.x
When adding comments to schema then hudi_sync don't add it to the hive
table. Even when the feature is activate
```
+ spark3.2-comments.py 08_pyspark_lbc_implem_cluster.py
X
from pyspark.sql.types import StructType, StructField, StringType,
IntegerType
data = [
(1, "f5c2ebfd-f57b-4ff3-ac5c-f30674037b21", "A", "BC", "C"),
(2, "f5c2ebfd-f57b-4ff3-ac5c-f30674037b22", "A", "BC", "C"),
(3, "f5c2ebfd-f57b-4ff3-ac5c-f30674037b21", "A", "BC", "C"),
(4, "f5c2ebfd-f57b-4ff3-ac5c-f30674037b22", "A", "BC", "C"),
]
schema = StructType(
[
StructField("uuid", IntegerType(), True, {"comment": "foo bar"}), #
Added a comment metadata
StructField("user_id", StringType(), True),
StructField("kafka_date", StringType(), True),
StructField("ts", StringType(), True),
StructField("part", StringType(), True),
]
)
df = spark.createDataFrame(data=data, schema=schema)
tableName = "test_hudi_pyspark"
basePath = f"/tmp/{tableName}"
hudi_options = {
"hoodie.table.name": tableName,
"hoodie.datasource.hive_sync.sync_comment": "true", # Activated
sync_comment
"hoodie.datasource.write.recordkey.field": "uuid",
"hoodie.datasource.write.partitionpath.field": "part",
"hoodie.datasource.write.table.name": tableName,
"hoodie.datasource.write.operation": "upsert",
"hoodie.datasource.write.precombine.field": "ts",
"hoodie.upsert.shuffle.parallelism": 2,
"hoodie.insert.shuffle.parallelism": 2,
"hoodie.datasource.hive_sync.database": "default",
"hoodie.datasource.hive_sync.table": tableName,
"hoodie.datasource.hive_sync.mode": "hms",
"hoodie.datasource.hive_sync.enable": "true",
"hoodie.datasource.hive_sync.partition_fields": "part",
"hoodie.datasource.hive_sync.partition_extractor_class":
"org.apache.hudi.hive.MultiPartKeysValueExtractor",
}
(df.write.format("hudi").options(**hudi_options).mode("overwrite").save(basePath))
spark.sql("desc test_hudi_pyspark").show()
# Here the comment is missing
>>> spark.sql("desc test_hudi_pyspark").show()
+--------------------+---------+-------+
| col_name|data_type|comment|
+--------------------+---------+-------+
| _hoodie_commit_time| string| |
|_hoodie_commit_seqno| string| |
| _hoodie_record_key| string| |
|_hoodie_partition...| string| |
| _hoodie_file_name| string| |
| uuid| int| |
| user_id| string| |
| kafka_date| string| |
| ts| string| |
| part| string| |
| | | |
| # Partitioning| | |
| Part 0| part| |
+--------------------+---------+-------+
# Here the comment exists
>>> df.registerTempTable("foo")
>>> spark.sql("desc foo").show()
+----------+---------+-------+
| col_name|data_type|comment|
+----------+---------+-------+
| uuid| int|foo bar|
| user_id| string| null|
|kafka_date| string| null|
| ts| string| null|
| part| string| null|
+----------+---------+-------+
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
