[hudi] branch master updated: [HUDI-5632] Fix failure launching Spark jobs from hudi-cli-bundle (#7790)

[sivabalan]

This is an automated email from the ASF dual-hosted git repository.

sivabalan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new 22eab392fc7 [HUDI-5632] Fix failure launching Spark jobs from 
hudi-cli-bundle (#7790)
22eab392fc7 is described below

commit 22eab392fc7f21c0e8e6a14d50a43516c395f59b
Author: Y Ethan Guo <ethan.guoyi...@gmail.com>
AuthorDate: Mon Jan 30 07:54:40 2023 -0800

    [HUDI-5632] Fix failure launching Spark jobs from hudi-cli-bundle (#7790)
    
    - Ensures that Hudi CLI commands which require launching Spark can be 
executed with hudi-cli-bundle
---
 .../src/main/java/org/apache/hudi/cli/utils/SparkUtil.java   | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkUtil.java 
b/hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkUtil.java
index 1cffdf941f9..fd09a27271a 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkUtil.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkUtil.java
@@ -57,10 +57,18 @@ public class SparkUtil {
     }
 
     File libDirectory = new File(new File(currentJar).getParent(), "lib");
-    // This lib directory may be not required, such as providing libraries 
through a bundle jar
     if (libDirectory.exists()) {
+      // When directly using hudi-cli module, the jars under the lib directory
+      // generated by the compilation is required
       Arrays.stream(libDirectory.list()).forEach(library ->
-              sparkLauncher.addJar(new File(libDirectory, 
library).getAbsolutePath()));
+          sparkLauncher.addJar(new File(libDirectory, 
library).getAbsolutePath()));
+    } else {
+      // When using hudi-cli-bundle, we also need to add the hudi-spark*-bundle
+      // so that the Hudi Spark job can be launched
+      String sparkBundleJarPath = System.getenv("SPARK_BUNDLE_JAR");
+      if (!StringUtils.isNullOrEmpty(sparkBundleJarPath)) {
+        sparkLauncher.addJar(sparkBundleJarPath);
+      }
     }
     return sparkLauncher;
   }