This is an automated email from the ASF dual-hosted git repository. yihua pushed a commit to branch release-0.13.0 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 8ffc7ce3d8fe352490c376583c518c837343a71b Author: Y Ethan Guo <[email protected]> AuthorDate: Mon Jan 30 07:54:40 2023 -0800 [HUDI-5632] Fix failure launching Spark jobs from hudi-cli-bundle (#7790) - Ensures that Hudi CLI commands which require launching Spark can be executed with hudi-cli-bundle --- .../src/main/java/org/apache/hudi/cli/utils/SparkUtil.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkUtil.java b/hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkUtil.java index 1cffdf941f9..fd09a27271a 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkUtil.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkUtil.java @@ -57,10 +57,18 @@ public class SparkUtil { } File libDirectory = new File(new File(currentJar).getParent(), "lib"); - // This lib directory may be not required, such as providing libraries through a bundle jar if (libDirectory.exists()) { + // When directly using hudi-cli module, the jars under the lib directory + // generated by the compilation is required Arrays.stream(libDirectory.list()).forEach(library -> - sparkLauncher.addJar(new File(libDirectory, library).getAbsolutePath())); + sparkLauncher.addJar(new File(libDirectory, library).getAbsolutePath())); + } else { + // When using hudi-cli-bundle, we also need to add the hudi-spark*-bundle + // so that the Hudi Spark job can be launched + String sparkBundleJarPath = System.getenv("SPARK_BUNDLE_JAR"); + if (!StringUtils.isNullOrEmpty(sparkBundleJarPath)) { + sparkLauncher.addJar(sparkBundleJarPath); + } } return sparkLauncher; }
