deepakpanda93 commented on issue #14306:
URL: https://github.com/apache/hudi/issues/14306#issuecomment-3665233083
@ROOBALJINDAL Now I am able to reproduce the issue with `Pay: decimal(20,5)`
```
25/12/17 12:22:57 ERROR BaseSparkCommitActionExecutor: Error upserting
bucketType UPDATE for partition :0
org.apache.hudi.exception.HoodieException:
org.apache.avro.SchemaParseException: Can't redefine: Pay
at
org.apache.hudi.table.action.commit.HoodieMergeHelper.runMerge(HoodieMergeHelper.java:149)
at
org.apache.hudi.table.action.commit.BaseSparkCommitActionExecutor.handleUpdateInternal(BaseSparkCommitActionExecutor.java:387)
at
org.apache.hudi.table.action.commit.BaseSparkCommitActionExecutor.handleUpdate(BaseSparkCommitActionExecutor.java:369)
at
org.apache.hudi.table.action.commit.BaseSparkCommitActionExecutor.handleUpsertPartition(BaseSparkCommitActionExecutor.java:335)
at
org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1458)
at
org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1522)
at
org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1349)
at org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:378)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:329)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.avro.SchemaParseException: Can't redefine: Pay
at org.apache.avro.Schema$Names.put(Schema.java:1586)
at org.apache.avro.Schema$NamedSchema.writeNameRef(Schema.java:844)
at org.apache.avro.Schema$RecordSchema.toJson(Schema.java:1011)
```
Schema:
```
root
|-- _hoodie_commit_time: string (nullable = true)
|-- _hoodie_commit_seqno: string (nullable = true)
|-- _hoodie_record_key: string (nullable = true)
|-- _hoodie_partition_path: string (nullable = true)
|-- _hoodie_file_name: string (nullable = true)
|-- Services: struct (nullable = true)
| |-- BasisCalculation: struct (nullable = true)
| | |-- Pay: decimal(20,5) (nullable = true)
|-- TotalPay: struct (nullable = true)
| |-- Pay: struct (nullable = true)
| | |-- Rate: struct (nullable = true)
| | | |-- _value: double (nullable = true)
| | | |-- defvalue: string (nullable = true)
| | |-- Basis: struct (nullable = true)
| | | |-- _value: string (nullable = true)
| | | |-- defvalue: string (nullable = true)
|-- record_key: string (nullable = false)
```
I tested the same with Hudi 1.1.0, and it is working without any issues.
```
Schema:
root
|-- _hoodie_commit_time: string (nullable = true)
|-- _hoodie_commit_seqno: string (nullable = true)
|-- _hoodie_record_key: string (nullable = true)
|-- _hoodie_partition_path: string (nullable = true)
|-- _hoodie_file_name: string (nullable = true)
|-- Services: struct (nullable = true)
| |-- BasisCalculation: struct (nullable = true)
| | |-- Pay: decimal(20,5) (nullable = true)
|-- TotalPay: struct (nullable = true)
| |-- Pay: struct (nullable = true)
| | |-- Rate: struct (nullable = true)
| | | |-- _value: double (nullable = true)
| | | |-- defvalue: string (nullable = true)
| | |-- Basis: struct (nullable = true)
| | | |-- _value: string (nullable = true)
| | | |-- defvalue: string (nullable = true)
|-- record_key: string (nullable = false)
+-------------------+---------------------+------------------+----------------------+------------------------------------------------------------------------+-------------------------+----------------------------------+----------+
|_hoodie_commit_time|_hoodie_commit_seqno
|_hoodie_record_key|_hoodie_partition_path|_hoodie_file_name
|Services |TotalPay
|record_key|
+-------------------+---------------------+------------------+----------------------+------------------------------------------------------------------------+-------------------------+----------------------------------+----------+
|20251217124148081 |20251217124148081_0_0|1 |
|2503c99d-d6d8-4173-bd32-5811e9db0c57-0_0-47-82_20251217124148081.parquet|{{999999999999999.99999}}|{{{1.0E15,
null}, {twenty, null}}}|1 |
+-------------------+---------------------+------------------+----------------------+------------------------------------------------------------------------+-------------------------+----------------------------------+----------+
root@6ba4293d5eb5:/opt# ls -l /tmp/hudi_xml_bug_table_110/
total 856
-rw-r--r-- 1 root root 436705 Dec 17 12:41
2503c99d-d6d8-4173-bd32-5811e9db0c57-0_0-26-40_20251217124145257.parquet
-rw-r--r-- 1 root root 436705 Dec 17 12:41
2503c99d-d6d8-4173-bd32-5811e9db0c57-0_0-47-82_20251217124148081.parquet
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
