soumilshah1995 commented on issue #10752:
URL: https://github.com/apache/hudi/issues/10752#issuecomment-1966560956
Hey @ad1happy2go
### Code
```
import com.amazonaws.services.glue.GlueContext
import com.amazonaws.services.glue.MappingSpec
import com.amazonaws.services.glue.errors.CallSite
import com.amazonaws.services.glue.util.GlueArgParser
import com.amazonaws.services.glue.util.Job
import com.amazonaws.services.glue.util.JsonOptions
import org.apache.spark.SparkContext
import scala.collection.JavaConverters._
import org.apache.spark.sql.SparkSession
import org.apache.spark.api.java.JavaSparkContext
import org.apache.hudi.utilities.streamer.HoodieStreamer
import org.apache.hudi.utilities.streamer.SchedulerConfGenerator
import org.apache.hudi.utilities.UtilHelpers
import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
object GlueApp {
def main(sysArgs: Array[String]) {
val args = GlueArgParser.getResolvedOptions(sysArgs,
Seq("JOB_NAME").toArray)
val BUCKET = "XXX"
var config = Array(
"--source-class", "org.apache.hudi.utilities.sources.ParquetDFSSource",
"--source-ordering-field", "replicadmstimestamp",
s"--target-base-path", s"s3://$BUCKET/test/",
"--target-table", "invoice",
"--table-type" , "COPY_ON_WRITE",
"--hoodie-conf",
"hoodie.datasource.write.keygenerator.class=org.apache.hudi.keygen.SimpleKeyGenerator",
"--hoodie-conf", "hoodie.datasource.write.recordkey.field=invoiceid",
"--hoodie-conf",
"hoodie.datasource.write.partitionpath.field=destinationstate",
s"--hoodie-conf",
s"hoodie.streamer.source.dfs.root=s3://$BUCKET/testcases/test_silver/",
"--hoodie-conf",
"hoodie.datasource.write.precombine.field=replicadmstimestamp"
)
val cfg = HoodieStreamer.getConfig(config)
val additionalSparkConfigs =
SchedulerConfGenerator.getSparkSchedulingConfigs(cfg)
val jssc = UtilHelpers.buildSparkContext("delta-streamer-test", "jes",
additionalSparkConfigs)
val spark = jssc.sc
val glueContext: GlueContext = new GlueContext(spark)
Job.init(args("JOB_NAME"), glueContext, args.asJava)
try {
new HoodieStreamer(cfg, jssc).sync();
} finally {
jssc.stop();
}
Job.commit()
}
}
```
Sample dataset
Link
https://drive.google.com/drive/folders/1BwNEK649hErbsWcYLZhqCWnaXFX3mIsg?usp=share_link
## Jar
These are jar I use in job
```
jcommander-1.78.jar
hudi-spark3.3-bundle_2.12-0.14.0.jar
hudi-utilities-slim-bundle_2.12-0.14.0.jar
```
Error
Error Category: RESOURCE_NOT_FOUND_ERROR; Exception in User Class:
org.apache.hudi.exception.TableNotFoundException : Hoodie table not found in
path s3://apply-systems-qa/test/.hoodie
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
