This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new c18b366e3794 docs: update javadoc spark-submit examples to Spark 3.5 /
Scala 2.12 (#19171)
c18b366e3794 is described below
commit c18b366e3794a76aee2c61fd5b1be91126f55784
Author: Y Ethan Guo <[email protected]>
AuthorDate: Wed Jul 8 04:53:20 2026 -0700
docs: update javadoc spark-submit examples to Spark 3.5 / Scala 2.12
(#19171)
---
.../hudi/integ/testsuite/HoodieContinuousTestSuiteWriter.java | 3 +--
.../hudi/integ/testsuite/HoodieMultiWriterTestSuiteJob.java | 3 +--
.../integ/testsuite/SparkDataSourceContinuousIngestTool.java | 3 +--
.../java/org/apache/hudi/utilities/HoodieDataTableValidator.java | 5 ++---
.../java/org/apache/hudi/utilities/HoodieDropPartitionsTool.java | 9 ++++-----
.../src/main/java/org/apache/hudi/utilities/HoodieIndexer.java | 3 +--
.../org/apache/hudi/utilities/HoodieMetadataTableValidator.java | 5 ++---
.../src/main/java/org/apache/hudi/utilities/TableSizeStats.java | 3 +--
8 files changed, 13 insertions(+), 21 deletions(-)
diff --git
a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/HoodieContinuousTestSuiteWriter.java
b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/HoodieContinuousTestSuiteWriter.java
index d49ee7fa6365..535e4049f11e 100644
---
a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/HoodieContinuousTestSuiteWriter.java
+++
b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/HoodieContinuousTestSuiteWriter.java
@@ -40,9 +40,8 @@ import java.util.Properties;
/**
* Test suite Writer that assists in testing async table operations with
Deltastreamer continuous mode.
*
- * TODO: [HUDI-8294]
* Sample command
- * ./bin/spark-submit --packages org.apache.spark:spark-avro_2.11:2.4.4 \
+ * ./bin/spark-submit --packages org.apache.spark:spark-avro_2.12:3.5.5 \
* --conf spark.task.cpus=1 --conf spark.executor.cores=1 \
* --conf spark.task.maxFailures=100 \
* --conf spark.memory.fraction=0.4 \
diff --git
a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/HoodieMultiWriterTestSuiteJob.java
b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/HoodieMultiWriterTestSuiteJob.java
index 96728d6853de..0a5485345f55 100644
---
a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/HoodieMultiWriterTestSuiteJob.java
+++
b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/HoodieMultiWriterTestSuiteJob.java
@@ -51,10 +51,9 @@ import java.util.concurrent.atomic.AtomicInteger;
* Writer 1 DeltaStreamer ingesting data into partitions 0 to 10, Writer 2
Spark datasource ingesting data into partitions 100 to 110.
* Multiple spark datasource writers, each writing to exclusive set of
partitions.
*
- * TODO: [HUDI-8294]
* Example command
* spark-submit
- * --packages org.apache.spark:spark-avro_2.11:2.4.0
+ * --packages org.apache.spark:spark-avro_2.12:3.5.5
* --conf spark.task.cpus=3
* --conf spark.executor.cores=3
* --conf spark.task.maxFailures=100
diff --git
a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/SparkDataSourceContinuousIngestTool.java
b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/SparkDataSourceContinuousIngestTool.java
index d8ebc8fe5909..6d91fa88ca9c 100644
---
a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/SparkDataSourceContinuousIngestTool.java
+++
b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/SparkDataSourceContinuousIngestTool.java
@@ -43,8 +43,7 @@ import java.util.Map;
/**
* Sample command
*
- * TODO: [HUDI-8294]
- * ./bin/spark-submit --packages org.apache.spark:spark-avro_2.11:2.4.4
--driver-memory 4g --executor-memory 4g \
+ * ./bin/spark-submit --packages org.apache.spark:spark-avro_2.12:3.5.5
--driver-memory 4g --executor-memory 4g \
* --conf spark.serializer=org.apache.spark.serializer.KryoSerializer --conf
spark.sql.catalogImplementation=hive \
* --class org.apache.hudi.integ.testsuite.SparkDSContinuousIngestTool \
*
${HUDI_ROOT_DIR}/packaging/hudi-integ-test-bundle/target/hudi-integ-test-bundle-0.11.0-SNAPSHOT.jar
\
diff --git
a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDataTableValidator.java
b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDataTableValidator.java
index 79ca3b647194..ed689022da64 100644
---
a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDataTableValidator.java
+++
b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDataTableValidator.java
@@ -61,7 +61,6 @@ import static
org.apache.hudi.common.table.timeline.InstantComparison.LESSER_THA
import static
org.apache.hudi.common.table.timeline.InstantComparison.compareTimestamps;
/**
- * TODO: [HUDI-8294]
* A validator with spark-submit to ensure there are no dangling data files in
the data table.
* No data files found for commits prior to active timeline.
* No extra data files found for completed commits more than whats present in
commit metadata.
@@ -76,7 +75,7 @@ import static
org.apache.hudi.common.table.timeline.InstantComparison.compareTim
* --master spark://xxxx:7077 \
* --driver-memory 1g \
* --executor-memory 1g \
- *
$HUDI_DIR/hudi/packaging/hudi-utilities-bundle/target/hudi-utilities-bundle_2.11-0.11.0-SNAPSHOT.jar
\
+ *
$HUDI_DIR/hudi/packaging/hudi-utilities-bundle/target/hudi-utilities-bundle_2.12-1.3.0-SNAPSHOT.jar
\
* --base-path basePath
* ```
*
@@ -91,7 +90,7 @@ import static
org.apache.hudi.common.table.timeline.InstantComparison.compareTim
* --master spark://xxxx:7077 \
* --driver-memory 1g \
* --executor-memory 1g \
- *
$HUDI_DIR/hudi/packaging/hudi-utilities-bundle/target/hudi-utilities-bundle_2.11-0.11.0-SNAPSHOT.jar
\
+ *
$HUDI_DIR/hudi/packaging/hudi-utilities-bundle/target/hudi-utilities-bundle_2.12-1.3.0-SNAPSHOT.jar
\
* --base-path basePath
* --continuous \
* --min-validate-interval-seconds 60
diff --git
a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDropPartitionsTool.java
b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDropPartitionsTool.java
index b9dadae7d87a..d945203e59ef 100644
---
a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDropPartitionsTool.java
+++
b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDropPartitionsTool.java
@@ -57,7 +57,6 @@ import java.util.stream.Collectors;
import scala.Tuple2;
/**
- * TODO: [HUDI-8294]
* A tool with spark-submit to drop Hudi table partitions.
*
* <p>
@@ -65,11 +64,11 @@ import scala.Tuple2;
* ```
* spark-submit \
* --class org.apache.hudi.utilities.HoodieDropPartitionsTool \
- * --packages org.apache.spark:spark-avro_2.11:2.4.4 \
+ * --packages org.apache.spark:spark-avro_2.12:3.5.5 \
* --master local[*]
* --driver-memory 1g \
* --executor-memory 1g \
- *
$HUDI_DIR/hudi/packaging/hudi-utilities-bundle/target/hudi-utilities-bundle_2.11-0.11.0-SNAPSHOT.jar
\
+ *
$HUDI_DIR/hudi/packaging/hudi-utilities-bundle/target/hudi-utilities-bundle_2.12-1.3.0-SNAPSHOT.jar
\
* --base-path basePath \
* --table-name tableName \
* --mode dry_run \
@@ -87,11 +86,11 @@ import scala.Tuple2;
* ```
* spark-submit \
* --class org.apache.hudi.utilities.HoodieDropPartitionsTool \
- * --packages org.apache.spark:spark-avro_2.11:2.4.4 \
+ * --packages org.apache.spark:spark-avro_2.12:3.5.5 \
* --master local[*]
* --driver-memory 1g \
* --executor-memory 1g \
- *
$HUDI_DIR/hudi/packaging/hudi-utilities-bundle/target/hudi-utilities-bundle_2.11-0.11.0-SNAPSHOT.jar
\
+ *
$HUDI_DIR/hudi/packaging/hudi-utilities-bundle/target/hudi-utilities-bundle_2.12-1.3.0-SNAPSHOT.jar
\
* --base-path basePath \
* --table-name tableName \
* --mode delete \
diff --git
a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieIndexer.java
b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieIndexer.java
index 2c7bab98900f..a107425e2665 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieIndexer.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieIndexer.java
@@ -62,14 +62,13 @@ import static
org.apache.hudi.utilities.UtilHelpers.SCHEDULE;
import static org.apache.hudi.utilities.UtilHelpers.SCHEDULE_AND_EXECUTE;
/**
- * TODO: [HUDI-8294]
* A tool to run metadata indexing asynchronously.
* <p>
* Example command (assuming indexer.properties contains related index
configs, see {@link org.apache.hudi.common.config.HoodieMetadataConfig} for
configs):
* <p>
* spark-submit \
* --class org.apache.hudi.utilities.HoodieIndexer \
- *
/path/to/hudi/packaging/hudi-utilities-bundle/target/hudi-utilities-bundle_2.11-0.11.0-SNAPSHOT.jar
\
+ *
/path/to/hudi/packaging/hudi-utilities-bundle/target/hudi-utilities-bundle_2.12-1.3.0-SNAPSHOT.jar
\
* --props /path/to/indexer.properties \
* --mode scheduleAndExecute \
* --base-path /tmp/hudi_trips_cow \
diff --git
a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java
b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java
index cac49c783b6e..3f28b4f14187 100644
---
a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java
+++
b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java
@@ -150,7 +150,6 @@ import static
org.apache.hudi.metadata.HoodieTableMetadataUtil.getLocationFromRe
import static
org.apache.hudi.metadata.HoodieTableMetadataUtil.getLogFileColumnRangeMetadata;
/**
- * TODO: [HUDI-8294]
* A validator with spark-submit to compare information, such as partitions,
file listing, index, etc.,
* between metadata table and filesystem.
* <p>
@@ -177,7 +176,7 @@ import static
org.apache.hudi.metadata.HoodieTableMetadataUtil.getLogFileColumnR
* --master spark://xxxx:7077 \
* --driver-memory 1g \
* --executor-memory 1g \
- *
$HUDI_DIR/packaging/hudi-utilities-bundle/target/hudi-utilities-bundle_2.11-0.13.0-SNAPSHOT.jar
\
+ *
$HUDI_DIR/packaging/hudi-utilities-bundle/target/hudi-utilities-bundle_2.12-1.3.0-SNAPSHOT.jar
\
* --base-path basePath \
* --validate-latest-file-slices \
* --validate-latest-base-files \
@@ -195,7 +194,7 @@ import static
org.apache.hudi.metadata.HoodieTableMetadataUtil.getLogFileColumnR
* --master spark://xxxx:7077 \
* --driver-memory 1g \
* --executor-memory 1g \
- *
$HUDI_DIR/packaging/hudi-utilities-bundle/target/hudi-utilities-bundle_2.11-0.13.0-SNAPSHOT.jar
\
+ *
$HUDI_DIR/packaging/hudi-utilities-bundle/target/hudi-utilities-bundle_2.12-1.3.0-SNAPSHOT.jar
\
* --base-path basePath \
* --validate-latest-file-slices \
* --validate-latest-base-files \
diff --git
a/hudi-utilities/src/main/java/org/apache/hudi/utilities/TableSizeStats.java
b/hudi-utilities/src/main/java/org/apache/hudi/utilities/TableSizeStats.java
index d017574bda1f..9a5bca95c318 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/TableSizeStats.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/TableSizeStats.java
@@ -68,7 +68,6 @@ import java.util.Set;
import java.util.stream.Collectors;
/**
- * TODO: [HUDI-8294]
* This class provides file size updates for the latest files that hudi is
consuming. These stats are at table level by default, but
* specifying --enable-partition-stats will also show stats at the partition
level. If a start date (--start-date parameter) and/or
* end date (--end-date parameter) are specified, stats are based on files
that were modified in the half-open interval
@@ -92,7 +91,7 @@ import java.util.stream.Collectors;
* Sample spark-submit command:
* ./bin/spark-submit \
* --class org.apache.hudi.utilities.TableSizeStats \
- *
$HUDI_DIR/packaging/hudi-utilities-bundle/target/hudi-utilities-bundle_2.11-0.14.0-SNAPSHOT.jar
\
+ *
$HUDI_DIR/packaging/hudi-utilities-bundle/target/hudi-utilities-bundle_2.12-1.3.0-SNAPSHOT.jar
\
* --base-path <base-path> \
* --num-days <number-of-days>
*/
