kapjoshi-cisco commented on issue #5636:
URL: https://github.com/apache/hudi/issues/5636#issuecomment-1135518894
~> Thank you this works~
Sorry there as an error handling section, which made it look like the job
passed, its still failing with the below error
Jars used:
1. s3://hudi-jars/jars/hudi-spark3-bundle_2.12-0.11.0.jar, # using 0.11.0
version
2. s3://hudi-jars/jars/spark-avro_2.12-3.1.2.jar,
3. s3://hudi-jars/jars/calcite-core-1.16.0.jar
```bash
java.lang.NoClassDefFoundError:
org/apache/parquet/schema/LogicalTypeAnnotation
at
org.apache.hudi.common.table.TableSchemaResolver.convertParquetSchemaToAvro(TableSchemaResolver.java:340)
at
org.apache.hudi.common.table.TableSchemaResolver.getTableAvroSchemaFromDataFile(TableSchemaResolver.java:148)
at
org.apache.hudi.common.table.TableSchemaResolver.hasOperationField(TableSchemaResolver.java:565)
at
org.apache.hudi.common.table.TableSchemaResolver.<init>(TableSchemaResolver.java:82)
at
org.apache.hudi.HoodieBaseRelation.x$1$lzycompute(HoodieBaseRelation.scala:126)
at org.apache.hudi.HoodieBaseRelation.x$1(HoodieBaseRelation.scala:125)
at
org.apache.hudi.HoodieBaseRelation.tableAvroSchema$lzycompute(HoodieBaseRelation.scala:125)
at
org.apache.hudi.HoodieBaseRelation.tableAvroSchema(HoodieBaseRelation.scala:125)
at org.apache.hudi.HoodieBaseRelation.<init>(HoodieBaseRelation.scala:147)
at org.apache.hudi.BaseFileOnlyRelation.<init>(BaseFileOnlyRelation.scala:53)
at
org.apache.hudi.DefaultSource.resolveBaseFileOnlyRelation(DefaultSource.scala:217)
at
org.apache.hudi.DefaultSource.createRelation(DefaultSource.scala:113)
at org.apache.hudi.DefaultSource.createRelation(DefaultSource.scala:66)
at
org.apache.spark.sql.execution.datasources.DataSource.resolveRelation(DataSource.scala:354)
at
org.apache.spark.sql.DataFrameReader.loadV1Source(DataFrameReader.scala:326)
at
org.apache.spark.sql.DataFrameReader.$anonfun$load$3(DataFrameReader.scala:308)
at scala.Option.getOrElse(Option.scala:189)
at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:308)
at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:240)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(Delegating | "error found
while loading data from path s3://hudi-bucket/nytaxis/*:" An error occurred
while calling o126.load. : java.lang.NoClassDefFoundError:
org/apache/parquet/schema/LogicalTypeAnnotation
at
org.apache.hudi.common.table.TableSchemaResolver.convertParquetSchemaToAvro(TableSchemaResolver.java:340)
at
org.apache.hudi.common.table.TableSchemaResolver.getTableAvroSchemaFromDataFile(TableSchemaResolver.java:148)
at
org.apache.hudi.common.table.TableSchemaResolver.hasOperationField(TableSchemaResolver.java:565)
at
org.apache.hudi.common.table.TableSchemaResolver.<init>(TableSchemaResolver.java:82)
at
org.apache.hudi.HoodieBaseRelation.x$1$lzycompute(HoodieBaseRelation.scala:126)
at org.apache.hudi.HoodieBaseRelation.x$1(HoodieBaseRelation.scala:125)
at
org.apache.hudi.HoodieBaseRelation.tableAvroSchema$lzycompute(HoodieBaseRelation.scala:125)
at
org.apache.hudi.HoodieBaseRelation.tableAvroSchema(HoodieBaseRelation.scala:125)
at org.apache.hudi.HoodieBaseRelation.<init>(HoodieBaseRelation.scala:147)
at
org.apache.hudi.BaseFileOnlyRelation.<init>(BaseFileOnlyRelation.scala:53)
at
org.apache.hudi.DefaultSource.resolveBaseFileOnlyRelation(DefaultSource.scala:217)
at org.apache.hudi.DefaultSource.createRelation(DefaultSource.scala:113)
at org.apache.hudi.DefaultSource.createRelation(DefaultSource.scala:66)
at
org.apache.spark.sql.execution.datasources.DataSource.resolveRelation(DataSource.scala:354)
at
org.apache.spark.sql.DataFrameReader.loadV1Source(DataFrameReader.scala:326)
at
org.apache.spark.sql.DataFrameReader.$anonfun$load$3(DataFrameReader.scala:308)
at scala.Option.getOrElse(Option.scala:189) at
org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:308)
at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:240)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(Delegating
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
