tjtoll commented on issue #5636:
URL: https://github.com/apache/hudi/issues/5636#issuecomment-1135737992
Since you went to .11 you might need this specific jar instead:
hudi-spark3.1-bundle_2.12-0.11.0.jar
Glue is spark 3.1 and I’m guessing the jar you have is 3.2
> On May 24, 2022, at 3:38 AM, Kapil Kumar Joshi ***@***.***> wrote:
>
>
> > Thank you this works
>
> Sorry there as an error handling section, which made it look like the job
passed, its still failing with the below error
>
> Jars used:
>
> s3://hudi-jars/jars/hudi-spark3-bundle_2.12-0.11.0.jar, # using 0.11.0
version
> s3://hudi-jars/jars/spark-avro_2.12-3.1.2.jar,
> s3://hudi-jars/jars/calcite-core-1.16.0.jar
> java.lang.NoClassDefFoundError:
org/apache/parquet/schema/LogicalTypeAnnotation
> at
org.apache.hudi.common.table.TableSchemaResolver.convertParquetSchemaToAvro(TableSchemaResolver.java:340)
> at
org.apache.hudi.common.table.TableSchemaResolver.getTableAvroSchemaFromDataFile(TableSchemaResolver.java:148)
at
org.apache.hudi.common.table.TableSchemaResolver.hasOperationField(TableSchemaResolver.java:565)
> at
org.apache.hudi.common.table.TableSchemaResolver.<init>(TableSchemaResolver.java:82)
> at
org.apache.hudi.HoodieBaseRelation.x$1$lzycompute(HoodieBaseRelation.scala:126)
> at org.apache.hudi.HoodieBaseRelation.x$1(HoodieBaseRelation.scala:125)
> at
org.apache.hudi.HoodieBaseRelation.tableAvroSchema$lzycompute(HoodieBaseRelation.scala:125)
> at
org.apache.hudi.HoodieBaseRelation.tableAvroSchema(HoodieBaseRelation.scala:125)
> at org.apache.hudi.HoodieBaseRelation.<init>(HoodieBaseRelation.scala:147)
at org.apache.hudi.BaseFileOnlyRelation.<init>(BaseFileOnlyRelation.scala:53)
> at
org.apache.hudi.DefaultSource.resolveBaseFileOnlyRelation(DefaultSource.scala:217)
> at
org.apache.hudi.DefaultSource.createRelation(DefaultSource.scala:113)
> at org.apache.hudi.DefaultSource.createRelation(DefaultSource.scala:66)
> at
org.apache.spark.sql.execution.datasources.DataSource.resolveRelation(DataSource.scala:354)
> at
org.apache.spark.sql.DataFrameReader.loadV1Source(DataFrameReader.scala:326)
> at
org.apache.spark.sql.DataFrameReader.$anonfun$load$3(DataFrameReader.scala:308)
> at scala.Option.getOrElse(Option.scala:189)
> at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:308)
> at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:240)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(Delegating | "error
found while loading data from path s3://hudi-bucket/nytaxis/*:" An error
occurred while calling o126.load. : java.lang.NoClassDefFoundError:
org/apache/parquet/schema/LogicalTypeAnnotation
> at
org.apache.hudi.common.table.TableSchemaResolver.convertParquetSchemaToAvro(TableSchemaResolver.java:340)
> at
org.apache.hudi.common.table.TableSchemaResolver.getTableAvroSchemaFromDataFile(TableSchemaResolver.java:148)
at
org.apache.hudi.common.table.TableSchemaResolver.hasOperationField(TableSchemaResolver.java:565)
> at
org.apache.hudi.common.table.TableSchemaResolver.<init>(TableSchemaResolver.java:82)
> at
org.apache.hudi.HoodieBaseRelation.x$1$lzycompute(HoodieBaseRelation.scala:126)
> at org.apache.hudi.HoodieBaseRelation.x$1(HoodieBaseRelation.scala:125)
> at
org.apache.hudi.HoodieBaseRelation.tableAvroSchema$lzycompute(HoodieBaseRelation.scala:125)
> at
org.apache.hudi.HoodieBaseRelation.tableAvroSchema(HoodieBaseRelation.scala:125)
> at org.apache.hudi.HoodieBaseRelation.<init>(HoodieBaseRelation.scala:147)
> at
org.apache.hudi.BaseFileOnlyRelation.<init>(BaseFileOnlyRelation.scala:53)
> at
org.apache.hudi.DefaultSource.resolveBaseFileOnlyRelation(DefaultSource.scala:217)
> at org.apache.hudi.DefaultSource.createRelation(DefaultSource.scala:113)
> at org.apache.hudi.DefaultSource.createRelation(DefaultSource.scala:66)
> at
org.apache.spark.sql.execution.datasources.DataSource.resolveRelation(DataSource.scala:354)
> at
org.apache.spark.sql.DataFrameReader.loadV1Source(DataFrameReader.scala:326)
> at
org.apache.spark.sql.DataFrameReader.$anonfun$load$3(DataFrameReader.scala:308)
> at scala.Option.getOrElse(Option.scala:189) at
org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:308)
> at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:240)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(Delegating
>
>
>
>
> —
> Reply to this email directly, view it on GitHub, or unsubscribe.
> You are receiving this because you commented.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
