xuzifu666 opened a new issue #3688:
URL: https://github.com/apache/hudi/issues/3688
**spark on hudi metadata key length <= 0, but data primary key is not "" or
null，error messages are in attatchment**
log as ：
Caused by: java.util.concurrent.ExecutionException:
org.apache.hudi.exception.HoodieUpsertException: Failed to merge old record
into new file for key from old file
s3://bigdata-oneentity-dw-cn-northwest-1/prod/hudi/ods/ods_rds_account_db_user_internal_id/.hoodie/metadata/files/a13cba6e-df5b-4999-bb32-18d0f6cbed1e-0_0-48-4136_20210906020857001.hfile
to new file
s3://bigdata-oneentity-dw-cn-northwest-1/prod/hudi/ods/ods_rds_account_db_user_internal_id/.hoodie/metadata/files/a13cba6e-df5b-4999-bb32-18d0f6cbed1e-0_0-48-4290_20210918044912001.hfile
with writerSchema {
"type" : "record",
"name" : "HoodieMetadataRecord",
"namespace" : "org.apache.hudi.avro.model",
"doc" : "A record saved within the Metadata Table",
"fields" : [ {
"name" : "_hoodie_commit_time",
"type" : [ "null", "string" ],
"doc" : "",
"default" : null
}, {
"name" : "_hoodie_commit_seqno",
"type" : [ "null", "string" ],
"doc" : "",
"default" : null
}, {
"name" : "_hoodie_record_key",
"type" : [ "null", "string" ],
"doc" : "",
"default" : null
}, {
"name" : "_hoodie_partition_path",
"type" : [ "null", "string" ],
"doc" : "",
"default" : null
}, {
"name" : "_hoodie_file_name",
"type" : [ "null", "string" ],
"doc" : "",
"default" : null
}, {
"name" : "key",
"type" : {
"type" : "string",
"avro.java.string" : "String"
}
}, {
"name" : "type",
"type" : "int",
"doc" : "Type of the metadata record"
}, {
"name" : "filesystemMetadata",
"type" : [ "null", {
"type" : "map",
"values" : {
"type" : "record",
"name" : "HoodieMetadataFileInfo",
"fields" : [ {
"name" : "size",
"type" : "long",
"doc" : "Size of the file"
}, {
"name" : "isDeleted",
"type" : "boolean",
"doc" : "True if this file has been deleted"
} ]
},
"avro.java.string" : "String"
} ],
"doc" : "Contains information about partitions and files within the
dataset"
} ]
}
at java.util.concurrent.FutureTask.report(FutureTask.java:122)
at java.util.concurrent.FutureTask.get(FutureTask.java:192)
at
org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.execute(BoundedInMemoryExecutor.java:141)
... 29 more
Caused by: org.apache.hudi.exception.HoodieUpsertException: Failed to merge
old record into new file for key from old file
s3://bigdata-oneentity-dw-cn-northwest-1/prod/hudi/ods/ods_rds_account_db_user_internal_id/.hoodie/metadata/files/a13cba6e-df5b-4999-bb32-18d0f6cbed1e-0_0-48-4136_20210906020857001.hfile
to new file
s3://bigdata-oneentity-dw-cn-northwest-1/prod/hudi/ods/ods_rds_account_db_user_internal_id/.hoodie/metadata/files/a13cba6e-df5b-4999-bb32-18d0f6cbed1e-0_0-48-4290_20210918044912001.hfile
with writerSchema {
"type" : "record",
"name" : "HoodieMetadataRecord",
"namespace" : "org.apache.hudi.avro.model",
"doc" : "A record saved within the Metadata Table",
"fields" : [ {
"name" : "_hoodie_commit_time",
"type" : [ "null", "string" ],
"doc" : "",
"default" : null
}, {
"name" : "_hoodie_commit_seqno",
"type" : [ "null", "string" ],
"doc" : "",
"default" : null
}, {
"name" : "_hoodie_record_key",
"type" : [ "null", "string" ],
"doc" : "",
"default" : null
}, {
"name" : "_hoodie_partition_path",
"type" : [ "null", "string" ],
"doc" : "",
"default" : null
}, {
"name" : "_hoodie_file_name",
"type" : [ "null", "string" ],
"doc" : "",
"default" : null
}, {
"name" : "key",
"type" : {
"type" : "string",
"avro.java.string" : "String"
}
}, {
"name" : "type",
"type" : "int",
"doc" : "Type of the metadata record"
}, {
"name" : "filesystemMetadata",
"type" : [ "null", {
"type" : "map",
"values" : {
"type" : "record",
"name" : "HoodieMetadataFileInfo",
"fields" : [ {
"name" : "size",
"type" : "long",
"doc" : "Size of the file"
}, {
"name" : "isDeleted",
"type" : "boolean",
"doc" : "True if this file has been deleted"
} ]
},
"avro.java.string" : "String"
} ],
"doc" : "Contains information about partitions and files within the
dataset"
} ]
}
at
org.apache.hudi.io.HoodieMergeHandle.write(HoodieMergeHandle.java:337)
at
org.apache.hudi.io.HoodieSortedMergeHandle.write(HoodieSortedMergeHandle.java:104)
at
org.apache.hudi.table.action.commit.AbstractMergeHelper$UpdateHandler.consumeOneRecord(AbstractMergeHelper.java:122)
at
org.apache.hudi.table.action.commit.AbstractMergeHelper$UpdateHandler.consumeOneRecord(AbstractMergeHelper.java:112)
at
org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer.consume(BoundedInMemoryQueueConsumer.java:37)
at
org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$2(BoundedInMemoryExecutor.java:121)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
... 3 more
Caused by: java.lang.IllegalArgumentException: key length must be > 0
at org.apache.hadoop.util.bloom.HashFunction.hash(HashFunction.java:114)
at org.apache.hadoop.util.bloom.BloomFilter.add(BloomFilter.java:122)
at
org.apache.hudi.common.bloom.InternalDynamicBloomFilter.add(InternalDynamicBloomFilter.java:94)
at
org.apache.hudi.common.bloom.HoodieDynamicBoundedBloomFilter.add(HoodieDynamicBoundedBloomFilter.java:81)
at
org.apache.hudi.io.storage.HoodieHFileWriter.writeAvro(HoodieHFileWriter.java:119)
at
org.apache.hudi.io.HoodieMergeHandle.write(HoodieMergeHandle.java:332)
... 9 more
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
