[
https://issues.apache.org/jira/browse/HUDI-9269?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
voon updated HUDI-9269:
-----------------------
Description:
`isTightBound` is field that was added to the hudi metadata payload.
When is created using 0.15.x with colstats enabled and read with a 1.0.x reader
an error will be thrown as the reader is not backwards compatible with the
following error:
When reading *colstats* index:
{code:java}
Caused by: org.apache.hudi.exception.HoodieException: Unable to instantiate
payload class
at
org.apache.hudi.common.util.HoodieRecordUtils.loadPayload(HoodieRecordUtils.java:114)
at
org.apache.hudi.common.util.SpillableMapUtils.convertToHoodieRecordPayload(SpillableMapUtils.java:155)
at
org.apache.hudi.metadata.HoodieBackedTableMetadata.composeRecord(HoodieBackedTableMetadata.java:560)
at
org.apache.hudi.metadata.HoodieBackedTableMetadata.lambda$fetchBaseFileRecordsByKeys$11(HoodieBackedTableMetadata.java:435)
at
java.base/java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:215)
at java.base/java.util.Iterator.forEachRemaining(Iterator.java:133)
at
java.base/java.util.Spliterators$IteratorSpliterator.forEachRemaining(Spliterators.java:1939)
at
java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:570)
at
java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:560)
at
java.base/java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:921)
at
java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:265)
at
java.base/java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:727)
at
org.apache.hudi.metadata.HoodieBackedTableMetadata.fetchBaseFileRecordsByKeys(HoodieBackedTableMetadata.java:437)
at
org.apache.hudi.metadata.HoodieBackedTableMetadata.readFromBaseAndMergeWithLogRecords(HoodieBackedTableMetadata.java:402)
at
org.apache.hudi.metadata.HoodieBackedTableMetadata.lambda$getRecordsByKeyPrefixes$7539c171$1(HoodieBackedTableMetadata.java:237)
at
org.apache.hudi.common.function.FunctionWrapper.lambda$throwingMapWrapper$0(FunctionWrapper.java:38)
... 27 more
Caused by: java.lang.reflect.InvocationTargetException
at
java.base/jdk.internal.reflect.DirectConstructorHandleAccessor.newInstance(DirectConstructorHandleAccessor.java:74)
at
java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:501)
at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:485)
at
org.apache.hudi.common.util.HoodieRecordUtils.loadPayload(HoodieRecordUtils.java:112)
... 42 more
Caused by: org.apache.avro.AvroRuntimeException: Not a valid schema field:
isTightBound
at org.apache.avro.generic.GenericData$Record.get(GenericData.java:282)
at
org.apache.hudi.metadata.MetadataPartitionType.constructColumnStatsMetadataPayload(MetadataPartitionType.java:318)
at
org.apache.hudi.metadata.MetadataPartitionType.access$200(MetadataPartitionType.java:97)
at
org.apache.hudi.metadata.MetadataPartitionType$2.constructMetadataPayload(MetadataPartitionType.java:122)
at
org.apache.hudi.metadata.HoodieMetadataPayload.<init>(HoodieMetadataPayload.java:204)
at
org.apache.hudi.metadata.HoodieMetadataPayload.<init>(HoodieMetadataPayload.java:190)
at
java.base/jdk.internal.reflect.DirectConstructorHandleAccessor.newInstance(DirectConstructorHandleAccessor.java:62)
... 45 more {code}
When reading RLI payload:
{code:java}
Caused by: org.apache.hudi.exception.HoodieException: Unable to instantiate
payload class
at
org.apache.hudi.common.util.HoodieRecordUtils.loadPayload(HoodieRecordUtils.java:114)
at
org.apache.hudi.common.util.SpillableMapUtils.convertToHoodieRecordPayload(SpillableMapUtils.java:155)
at
org.apache.hudi.metadata.HoodieBackedTableMetadata.composeRecord(HoodieBackedTableMetadata.java:560)
at
org.apache.hudi.metadata.HoodieBackedTableMetadata.lambda$fetchBaseFileAllRecordsByKeys$15(HoodieBackedTableMetadata.java:550)
at
java.base/java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:215)
at java.base/java.util.Iterator.forEachRemaining(Iterator.java:133)
at
java.base/java.util.Spliterators$IteratorSpliterator.forEachRemaining(Spliterators.java:1939)
at
java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:570)
at
java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:560)
at
java.base/java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:921)
at
java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:265)
at
java.base/java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:727)
at
org.apache.hudi.metadata.HoodieBackedTableMetadata.fetchBaseFileAllRecordsByKeys(HoodieBackedTableMetadata.java:552)
at
org.apache.hudi.metadata.HoodieBackedTableMetadata.readFromBaseAndMergeWithAllLogRecords(HoodieBackedTableMetadata.java:500)
at
org.apache.hudi.metadata.HoodieBackedTableMetadata.lookupAllKeysFromFileSlice(HoodieBackedTableMetadata.java:456)
at
org.apache.hudi.metadata.HoodieBackedTableMetadata.lambda$getAllRecordsByKeys$61b9eeed$1(HoodieBackedTableMetadata.java:327)
at
org.apache.hudi.common.function.FunctionWrapper.lambda$throwingMapWrapper$0(FunctionWrapper.java:38)
... 13 more
Caused by: java.lang.reflect.InvocationTargetException
at
java.base/jdk.internal.reflect.DirectConstructorHandleAccessor.newInstance(DirectConstructorHandleAccessor.java:74)
at
java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:501)
at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:485)
at
org.apache.hudi.common.util.HoodieRecordUtils.loadPayload(HoodieRecordUtils.java:112)
... 29 more
Caused by: org.apache.avro.AvroRuntimeException: Not a valid schema field:
position
at org.apache.avro.generic.GenericData$Record.get(GenericData.java:282)
at
org.apache.hudi.metadata.MetadataPartitionType$4.constructMetadataPayload(MetadataPartitionType.java:176)
at
org.apache.hudi.metadata.HoodieMetadataPayload.<init>(HoodieMetadataPayload.java:204)
at
org.apache.hudi.metadata.HoodieMetadataPayload.<init>(HoodieMetadataPayload.java:190)
at
java.base/jdk.internal.reflect.DirectConstructorHandleAccessor.newInstance(DirectConstructorHandleAccessor.java:62)
... 32 more {code}
was:
`isTightBound` is field that was added to the hudi metadata payload.
When is created using 0.15.x with colstats enabled and read with a 1.0.x reader
an error will be thrown as the reader is not backwards compatible with the
following error:
{code:java}
Caused by: org.apache.hudi.exception.HoodieException: Unable to instantiate
payload class
at
org.apache.hudi.common.util.HoodieRecordUtils.loadPayload(HoodieRecordUtils.java:114)
at
org.apache.hudi.common.util.SpillableMapUtils.convertToHoodieRecordPayload(SpillableMapUtils.java:155)
at
org.apache.hudi.metadata.HoodieBackedTableMetadata.composeRecord(HoodieBackedTableMetadata.java:560)
at
org.apache.hudi.metadata.HoodieBackedTableMetadata.lambda$fetchBaseFileRecordsByKeys$11(HoodieBackedTableMetadata.java:435)
at
java.base/java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:215)
at java.base/java.util.Iterator.forEachRemaining(Iterator.java:133)
at
java.base/java.util.Spliterators$IteratorSpliterator.forEachRemaining(Spliterators.java:1939)
at
java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:570)
at
java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:560)
at
java.base/java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:921)
at
java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:265)
at
java.base/java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:727)
at
org.apache.hudi.metadata.HoodieBackedTableMetadata.fetchBaseFileRecordsByKeys(HoodieBackedTableMetadata.java:437)
at
org.apache.hudi.metadata.HoodieBackedTableMetadata.readFromBaseAndMergeWithLogRecords(HoodieBackedTableMetadata.java:402)
at
org.apache.hudi.metadata.HoodieBackedTableMetadata.lambda$getRecordsByKeyPrefixes$7539c171$1(HoodieBackedTableMetadata.java:237)
at
org.apache.hudi.common.function.FunctionWrapper.lambda$throwingMapWrapper$0(FunctionWrapper.java:38)
... 27 more
Caused by: java.lang.reflect.InvocationTargetException
at
java.base/jdk.internal.reflect.DirectConstructorHandleAccessor.newInstance(DirectConstructorHandleAccessor.java:74)
at
java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:501)
at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:485)
at
org.apache.hudi.common.util.HoodieRecordUtils.loadPayload(HoodieRecordUtils.java:112)
... 42 more
Caused by: org.apache.avro.AvroRuntimeException: Not a valid schema field:
isTightBound
at org.apache.avro.generic.GenericData$Record.get(GenericData.java:282)
at
org.apache.hudi.metadata.MetadataPartitionType.constructColumnStatsMetadataPayload(MetadataPartitionType.java:318)
at
org.apache.hudi.metadata.MetadataPartitionType.access$200(MetadataPartitionType.java:97)
at
org.apache.hudi.metadata.MetadataPartitionType$2.constructMetadataPayload(MetadataPartitionType.java:122)
at
org.apache.hudi.metadata.HoodieMetadataPayload.<init>(HoodieMetadataPayload.java:204)
at
org.apache.hudi.metadata.HoodieMetadataPayload.<init>(HoodieMetadataPayload.java:190)
at
java.base/jdk.internal.reflect.DirectConstructorHandleAccessor.newInstance(DirectConstructorHandleAccessor.java:62)
... 45 more {code}
> HoodieRecordUtils fails to read payloads written by older tables
> ----------------------------------------------------------------
>
> Key: HUDI-9269
> URL: https://issues.apache.org/jira/browse/HUDI-9269
> Project: Apache Hudi
> Issue Type: Bug
> Affects Versions: 1.0.1
> Reporter: voon
> Priority: Major
>
> `isTightBound` is field that was added to the hudi metadata payload.
> When is created using 0.15.x with colstats enabled and read with a 1.0.x
> reader an error will be thrown as the reader is not backwards compatible with
> the following error:
>
> When reading *colstats* index:
> {code:java}
> Caused by: org.apache.hudi.exception.HoodieException: Unable to instantiate
> payload class
> at
> org.apache.hudi.common.util.HoodieRecordUtils.loadPayload(HoodieRecordUtils.java:114)
> at
> org.apache.hudi.common.util.SpillableMapUtils.convertToHoodieRecordPayload(SpillableMapUtils.java:155)
> at
> org.apache.hudi.metadata.HoodieBackedTableMetadata.composeRecord(HoodieBackedTableMetadata.java:560)
> at
> org.apache.hudi.metadata.HoodieBackedTableMetadata.lambda$fetchBaseFileRecordsByKeys$11(HoodieBackedTableMetadata.java:435)
> at
> java.base/java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:215)
> at java.base/java.util.Iterator.forEachRemaining(Iterator.java:133)
> at
> java.base/java.util.Spliterators$IteratorSpliterator.forEachRemaining(Spliterators.java:1939)
> at
> java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:570)
> at
> java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:560)
> at
> java.base/java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:921)
> at
> java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:265)
> at
> java.base/java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:727)
> at
> org.apache.hudi.metadata.HoodieBackedTableMetadata.fetchBaseFileRecordsByKeys(HoodieBackedTableMetadata.java:437)
> at
> org.apache.hudi.metadata.HoodieBackedTableMetadata.readFromBaseAndMergeWithLogRecords(HoodieBackedTableMetadata.java:402)
> at
> org.apache.hudi.metadata.HoodieBackedTableMetadata.lambda$getRecordsByKeyPrefixes$7539c171$1(HoodieBackedTableMetadata.java:237)
> at
> org.apache.hudi.common.function.FunctionWrapper.lambda$throwingMapWrapper$0(FunctionWrapper.java:38)
> ... 27 more
> Caused by: java.lang.reflect.InvocationTargetException
> at
> java.base/jdk.internal.reflect.DirectConstructorHandleAccessor.newInstance(DirectConstructorHandleAccessor.java:74)
> at
> java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:501)
> at
> java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:485)
> at
> org.apache.hudi.common.util.HoodieRecordUtils.loadPayload(HoodieRecordUtils.java:112)
> ... 42 more
> Caused by: org.apache.avro.AvroRuntimeException: Not a valid schema field:
> isTightBound
> at org.apache.avro.generic.GenericData$Record.get(GenericData.java:282)
> at
> org.apache.hudi.metadata.MetadataPartitionType.constructColumnStatsMetadataPayload(MetadataPartitionType.java:318)
> at
> org.apache.hudi.metadata.MetadataPartitionType.access$200(MetadataPartitionType.java:97)
> at
> org.apache.hudi.metadata.MetadataPartitionType$2.constructMetadataPayload(MetadataPartitionType.java:122)
> at
> org.apache.hudi.metadata.HoodieMetadataPayload.<init>(HoodieMetadataPayload.java:204)
> at
> org.apache.hudi.metadata.HoodieMetadataPayload.<init>(HoodieMetadataPayload.java:190)
> at
> java.base/jdk.internal.reflect.DirectConstructorHandleAccessor.newInstance(DirectConstructorHandleAccessor.java:62)
> ... 45 more {code}
>
> When reading RLI payload:
> {code:java}
> Caused by: org.apache.hudi.exception.HoodieException: Unable to instantiate
> payload class
> at
> org.apache.hudi.common.util.HoodieRecordUtils.loadPayload(HoodieRecordUtils.java:114)
> at
> org.apache.hudi.common.util.SpillableMapUtils.convertToHoodieRecordPayload(SpillableMapUtils.java:155)
> at
> org.apache.hudi.metadata.HoodieBackedTableMetadata.composeRecord(HoodieBackedTableMetadata.java:560)
> at
> org.apache.hudi.metadata.HoodieBackedTableMetadata.lambda$fetchBaseFileAllRecordsByKeys$15(HoodieBackedTableMetadata.java:550)
> at
> java.base/java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:215)
> at java.base/java.util.Iterator.forEachRemaining(Iterator.java:133)
> at
> java.base/java.util.Spliterators$IteratorSpliterator.forEachRemaining(Spliterators.java:1939)
> at
> java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:570)
> at
> java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:560)
> at
> java.base/java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:921)
> at
> java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:265)
> at
> java.base/java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:727)
> at
> org.apache.hudi.metadata.HoodieBackedTableMetadata.fetchBaseFileAllRecordsByKeys(HoodieBackedTableMetadata.java:552)
> at
> org.apache.hudi.metadata.HoodieBackedTableMetadata.readFromBaseAndMergeWithAllLogRecords(HoodieBackedTableMetadata.java:500)
> at
> org.apache.hudi.metadata.HoodieBackedTableMetadata.lookupAllKeysFromFileSlice(HoodieBackedTableMetadata.java:456)
> at
> org.apache.hudi.metadata.HoodieBackedTableMetadata.lambda$getAllRecordsByKeys$61b9eeed$1(HoodieBackedTableMetadata.java:327)
> at
> org.apache.hudi.common.function.FunctionWrapper.lambda$throwingMapWrapper$0(FunctionWrapper.java:38)
> ... 13 more
> Caused by: java.lang.reflect.InvocationTargetException
> at
> java.base/jdk.internal.reflect.DirectConstructorHandleAccessor.newInstance(DirectConstructorHandleAccessor.java:74)
> at
> java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:501)
> at
> java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:485)
> at
> org.apache.hudi.common.util.HoodieRecordUtils.loadPayload(HoodieRecordUtils.java:112)
> ... 29 more
> Caused by: org.apache.avro.AvroRuntimeException: Not a valid schema field:
> position
> at org.apache.avro.generic.GenericData$Record.get(GenericData.java:282)
> at
> org.apache.hudi.metadata.MetadataPartitionType$4.constructMetadataPayload(MetadataPartitionType.java:176)
> at
> org.apache.hudi.metadata.HoodieMetadataPayload.<init>(HoodieMetadataPayload.java:204)
> at
> org.apache.hudi.metadata.HoodieMetadataPayload.<init>(HoodieMetadataPayload.java:190)
> at
> java.base/jdk.internal.reflect.DirectConstructorHandleAccessor.newInstance(DirectConstructorHandleAccessor.java:62)
> ... 32 more {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
