[
https://issues.apache.org/jira/browse/HUDI-9093?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Mansi Patel reassigned HUDI-9093:
---------------------------------
Assignee: Mansi Patel
> HoodieClusteringJob has bean set two options with the same "-sc" parameter
> --------------------------------------------------------------------------
>
> Key: HUDI-9093
> URL: https://issues.apache.org/jira/browse/HUDI-9093
> Project: Apache Hudi
> Issue Type: Bug
> Reporter: Mansi Patel
> Assignee: Mansi Patel
> Priority: Major
> Labels: pull-request-available
> Fix For: 1.1.0
>
>
> {color:#1f2328}HoodieCompactor has bean set two options with the same "-sc"
> parameter.{color}
> *{color:#1f2328}Source Code:{color}*
> {code:java}
> @Parameter(names = {"--skip-clean", "-sc"}, description = "do not trigger
> clean after compaction", required = false)
> public Boolean skipClean = true;
> @Parameter(names = {"--schedule", "-sc"}, description = "Schedule
> compaction", required = false)
> public Boolean runSchedule = false;
> @Parameter(names = {"--mode", "-m"}, description = "Set job mode: Set
> \"schedule\" means make a compact plan; "
> + "Set \"execute\" means execute a compact plan at given instant
> which means --instant-time is needed here; "
> + "Set \"scheduleAndExecute\" means make a compact plan first and
> execute that plan immediately", required = false)
> {code}
> {color:#1f2328}{color}{color:#1f2328}[https://github.com/apache/hudi/blob/master/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCompactor.java#L97]{color}
> *{color:#1f2328}Spark-Submit command:{color}*
> {code:java}
> spark-submit --class org.apache.hudi.utilities.HoodieCompactor \
>
> ${HUDI_REPO}hudi/packaging/hudi-utilities-bundle/target/hudi-utilities-bundle_2.12-1.1.0-SNAPSHOT.jar
> \
> --jars
> ${HUDI_JARS}/hudi-spark3.4-bundle_2.12-1.0.0.jar,${HUDI_JARS}/hadoop-aws-3.3.4.jar,${HUDI_JARS}//aws-java-sdk-bundle-1.12.367.jar
> \
> --conf spark.hadoop.fs.s3a.endpoint=http://127.0.0.1:19000 \
> --conf spark.hadoop.fs.s3a.access.key=xxxxx \
> --conf spark.hadoop.fs.s3a.secret.key=xxxxx \
> --conf spark.hadoop.fs.s3a.path.style.access=true \
> --conf spark.hadoop.fs.s3a.connection.ssl.enable=false \
> --conf spark.hadoop.fs.s3a.impl=org.apache.hadoop.fs.s3a.S3AFileSystem \
> --base-path s3a://test/hudi-test-topic \
> --table-name hudi-test-topic \
> --schema-file s3a://test/schema.avsc \
> --instant-time 20250211112324137 \
> --parallelism 2 \
> --spark-memory 1g {code}
> {color:#1f2328}*Exception:*{color}
> {code:java}
> Exception in thread "main"
> org.apache.hudi.com.beust.jcommander.ParameterException: Found the option -sc
> multiple times
> at
> org.apache.hudi.com.beust.jcommander.JCommander.addDescription(JCommander.java:627)
> at
> org.apache.hudi.com.beust.jcommander.JCommander.createDescriptions(JCommander.java:594)
> at
> org.apache.hudi.com.beust.jcommander.JCommander.<init>(JCommander.java:249)
> at
> org.apache.hudi.utilities.HoodieCompactor.main(HoodieCompactor.java:173)
> at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method)
> at
> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
> at
> java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.base/java.lang.reflect.Method.invoke(Method.java:568)
> at
> org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
> at
> org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1020)
> at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:192)
> at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:215)
> at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)
> at
> org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1111)
> at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1120)
> at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
> 25/02/12 17:06:22 INFO ShutdownHookManager: Shutdown hook called
> 25/02/12 17:06:22 INFO ShutdownHookManager: Deleting directory
> /private/var/folders/vy/55g0v9hj7k30l58rx3crpgj40000gn/T/spark-91863f32-7ad6-47bc-9277-cc5eae852871
> {code}
> {color:#1f2328}*Issue Reference:*{color}
> {color:#1f2328}https://github.com/apache/hudi/issues/12832{color}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
