bamaer commented on code in PR #9595:
URL: https://github.com/apache/hop/pull/9595#discussion_r4653218664


##########
core/src/main/java/org/apache/hop/core/fatjar/NativeSparkFatJar.java:
##########
@@ -0,0 +1,303 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.core.fatjar;
+
+import java.io.File;
+import java.nio.file.Path;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import org.apache.hop.core.IProgressMonitor;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.logging.ILogChannel;
+import org.apache.hop.core.variables.IVariables;
+
+/**
+ * Which installed jars go into a fat jar for the native Spark engine (Spark 
4, Scala 2.13).
+ *
+ * <p>Two flavors:
+ *
+ * <ul>
+ *   <li><b>native</b>: leaves out Beam's Spark 3 / Scala 2.12 client 
classpath and keeps the Spark
+ *       4 runtime of {@code plugins/engines/spark/lib}.
+ *   <li><b>native-provided</b>: also leaves out every Spark, Scala and Hadoop 
client runtime jar,
+ *       because the cluster provides them. This is the flavor for a real 
cluster: standalone, YARN
+ *       (where {@code spark.yarn.jars} ships the engine's Spark runtime), 
Kubernetes and
+ *       Databricks.
+ * </ul>
+ *
+ * <p>Shared by the Spark engine (automatic fat jar), the Databricks 
deployment and {@code hop-conf
+ * --generate-fat-jar --fat-jar-engine=spark}.
+ */
+public final class NativeSparkFatJar {
+
+  /**
+   * Written by the engine plugin at build time: its Maven dependency tree, 
which tells what a Spark
+   * distribution of the engine's Spark version ships.
+   */
+  public static final String SPARK_DEPENDENCIES_FILE =
+      "plugins/engines/spark/spark-dependencies.txt";
+
+  /** Cache flavor of a native-provided fat jar: {@code 
hop-native-provided-<fingerprint>.jar}. */
+  public static final String FLAVOR_PROVIDED = "hop-native-provided";
+
+  /** Cache flavor of a native fat jar that embeds the Spark runtime. */
+  public static final String FLAVOR_NATIVE = "hop-native";
+
+  private NativeSparkFatJar() {}
+
+  /**
+   * @param provided true to leave out the Spark/Scala/Hadoop runtime the 
cluster provides
+   * @return the jar filenames of the installation in the working directory
+   */
+  public static List<String> findJarFilenames(boolean provided) {
+    return findJarFilenames(new File("."), provided);
+  }
+
+  /**
+   * @param installation the Hop installation folder
+   * @param provided true to leave out what the Spark cluster provides
+   */
+  public static List<String> findJarFilenames(File installation, boolean 
provided) {
+    Set<String> sparkArtifacts = provided ? 
sparkDistributionArtifacts(installation) : Set.of();
+    return InstalledJars.find(
+        installation,
+        file ->
+            isExcluded(file, provided)
+                || (!file.isDirectory() && isSparkDistributionJar(file, 
sparkArtifacts)));
+  }
+
+  /**
+   * The artifact ids a Spark distribution of the engine's Spark version 
ships, from the dependency
+   * tree the engine plugin writes at build time ({@value 
#SPARK_DEPENDENCIES_FILE}): everything
+   * under its {@code org.apache.spark} and {@code org.apache.hadoop} 
dependencies. Empty when the
+   * file is missing, for an engine plugin built before it existed.
+   */
+  static Set<String> sparkDistributionArtifacts(File installation) {
+    File tree = new File(installation, SPARK_DEPENDENCIES_FILE);
+    if (!tree.isFile()) {
+      return Set.of();
+    }
+    try {
+      return parseDistributionArtifacts(
+          java.nio.file.Files.readAllLines(tree.toPath(), 
java.nio.charset.StandardCharsets.UTF_8));
+    } catch (java.io.IOException e) {
+      return Set.of();
+    }
+  }
+
+  /**
+   * Parse {@code mvn dependency:tree} text output: the artifact ids of the 
top-level {@code
+   * org.apache.spark} and {@code org.apache.hadoop} dependencies and 
everything below them.
+   */
+  static Set<String> parseDistributionArtifacts(List<String> lines) {
+    Set<String> artifacts = new HashSet<>();
+    boolean inDistribution = false;
+    for (String line : lines) {
+      int start = 0;
+      while (start < line.length() && 
!Character.isLetterOrDigit(line.charAt(start))) {
+        start++;
+      }
+      if (start == 0 || start >= line.length()) {
+        continue; // the root (the engine plugin itself) or an empty line
+      }
+      String[] coordinates = line.substring(start).trim().split(":");
+      if (coordinates.length < 4) {
+        continue;
+      }
+      if (start == 3) {
+        // a direct dependency of the engine plugin: does a Spark distribution 
ship it?
+        inDistribution =
+            "org.apache.spark".equals(coordinates[0]) || 
"org.apache.hadoop".equals(coordinates[0]);
+      }
+      if (inDistribution) {
+        artifacts.add(coordinates[1]);
+      }
+    }
+    return artifacts;
+  }
+
+  /**
+   * A jar named after one of these artifacts: {@code 
<artifactId>-<version>[-classifier].jar}. The
+   * lakehouse connectors in the engine's lib/delta and lib/iceberg always 
stay: no Spark
+   * distribution ships them.
+   */
+  static boolean isSparkDistributionJar(File jar, Set<String> sparkArtifacts) {

Review Comment:
   **Blocker:** the provided jar now leaves out jars that the cluster doesn't 
provide. Jars are matched on artifactId only, against the Maven tree:
   - `plugins/transforms/script/lib/xz-25.0.4.jar` and `icu4j-25.0.4.jar` are 
GraalVM's shadowed copies (`org/graalvm/shadowed/...`). They are dropped 
because Spark ships `org.tukaani:xz` and `com.ibm.icu:icu4j`.
   - `lib/core/protobuf-java-4.34.1.jar` is dropped, but Spark 4.1.3 shades 
protobuf: its `jars/` folder has no `com/google/protobuf` classes. The gs:// 
file system (`google-cloud-storage`) and the Google Secret Manager resolver use 
them.
   
   Generate rows → Script (JavaScript) → Text file output on the cluster fails 
on every executor with this jar:
   ```
   NoClassDefFoundError: org/graalvm/shadowed/com/ibm/icu/text/DateFormat
   ```
   The same install with main's jar selection writes the 100 rows.
   
   Suggested fix: apply the dependency-tree exclusion to 
`plugins/engines/spark/lib` only. Those are the Spark runtime copies, and 
that's most of the saving (153 of the ~204 MB it removes, uncompressed). A unit 
test with a shadowed `xz` in a transform plugin and `protobuf-java` in 
`lib/core` would cover it.



##########
core/src/main/java/org/apache/hop/core/fatjar/FatJarCache.java:
##########
@@ -0,0 +1,266 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.core.fatjar;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.attribute.FileTime;
+import java.security.DigestInputStream;
+import java.security.MessageDigest;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.HexFormat;
+import java.util.List;
+import java.util.UUID;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hop.core.IProgressMonitor;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.logging.ILogChannel;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.i18n.BaseMessages;
+
+/**
+ * A local cache of fat jars, so a run does not rebuild a fat jar the 
installation already has.
+ *
+ * <p>A cached jar is named after its flavor and a fingerprint of its input 
jars (path, size and
+ * modification time): {@code <flavor>-<fingerprint>.jar}. Installing or 
upgrading a plugin changes
+ * the fingerprint, and the next run builds a new jar. Next to the jar a 
{@code .sha256} sidecar
+ * holds the SHA-256 of the jar contents, so a deployment can name and compare 
the jar it uploads
+ * without hashing a few hundred megabytes again.
+ *
+ * <p>The cache folder is {@value #VARIABLE_CACHE_FOLDER} (variable, system 
property or environment
+ * variable), or {@code hop-fat-jars} in the temporary folder. Jars of the 
same flavor that were not
+ * used for {@link #RETENTION} are removed after a build.
+ */
+public final class FatJarCache {
+
+  /** Folder of the fat jar cache. */
+  public static final String VARIABLE_CACHE_FOLDER = 
"HOP_FAT_JAR_CACHE_FOLDER";
+
+  public static final String SHA256_SIDECAR_SUFFIX = ".sha256";
+
+  /** Number of fingerprint hex characters in a cached jar name. */
+  static final int FINGERPRINT_LENGTH = 16;
+
+  /** Unused jars of a flavor older than this are removed after a new build. */
+  static final Duration RETENTION = Duration.ofDays(7);
+
+  private static final Object BUILD_LOCK = new Object();
+
+  private static final Class<?> PKG = FatJarCache.class;
+
+  private FatJarCache() {}
+
+  /**
+   * @param log log channel for the build
+   * @param variables resolves {@value #VARIABLE_CACHE_FOLDER}
+   * @param flavor jar name prefix, for example {@link 
NativeSparkFatJar#FLAVOR_PROVIDED}
+   * @param jarFilenames the jars to merge
+   * @return the cached fat jar for exactly these input jars
+   */
+  public static Path getOrBuild(
+      ILogChannel log, IVariables variables, String flavor, List<String> 
jarFilenames)
+      throws HopException {
+    return getOrBuild(log, variables, flavor, jarFilenames, null);
+  }
+
+  /**
+   * @param progressMonitor optional: follows a build and can cancel it; a 
cached jar is returned
+   *     without using it
+   * @see #getOrBuild(ILogChannel, IVariables, String, List)
+   */
+  public static Path getOrBuild(
+      ILogChannel log,
+      IVariables variables,
+      String flavor,
+      List<String> jarFilenames,
+      IProgressMonitor progressMonitor)
+      throws HopException {
+    Path folder = cacheFolder(variables);
+    String fingerprint = fingerprint(flavor, jarFilenames);
+    Path jar = folder.resolve(flavor + "-" + fingerprint + ".jar");
+
+    synchronized (BUILD_LOCK) {
+      if (Files.isRegularFile(jar) && Files.isRegularFile(sidecar(jar))) {
+        touch(jar);
+        log.logBasic("Using cached fat jar " + jar);
+        return jar;
+      }
+
+      long start = System.currentTimeMillis();
+      log.logBasic(
+          "Building fat jar "
+              + jar
+              + " from "
+              + jarFilenames.size()
+              + " jar files. This happens once per installation change.");
+      try {
+        Files.createDirectories(folder);
+      } catch (IOException e) {
+        throw new HopException("Unable to create the fat jar cache folder " + 
folder, e);
+      }
+      // Build next to the final name and move it in place, so a reader never 
sees half a jar.
+      Path temp = folder.resolve(jar.getFileName() + ".tmp-" + 
UUID.randomUUID());
+      try {
+        FatJarBuilder builder = new FatJarBuilder(log, variables, 
temp.toString(), jarFilenames);
+        builder.setProgressMonitor(progressMonitor);
+        builder.buildTargetJar();
+        if (progressMonitor != null) {
+          progressMonitor.subTask(BaseMessages.getString(PKG, 
"FatJarCache.Progress.Checksum"));
+        }
+        String sha = sha256Hex(temp);
+        Path tempSidecar = folder.resolve(temp.getFileName() + 
SHA256_SIDECAR_SUFFIX);
+        Files.writeString(tempSidecar, sha + "\n", StandardCharsets.UTF_8);
+        Files.move(temp, jar, StandardCopyOption.REPLACE_EXISTING, 
StandardCopyOption.ATOMIC_MOVE);
+        Files.move(
+            tempSidecar,
+            sidecar(jar),
+            StandardCopyOption.REPLACE_EXISTING,
+            StandardCopyOption.ATOMIC_MOVE);
+      } catch (IOException e) {
+        throw new HopException("Unable to store fat jar " + jar, e);
+      } finally {
+        deleteQuietly(temp);
+        deleteQuietly(folder.resolve(temp.getFileName() + 
SHA256_SIDECAR_SUFFIX));
+      }
+      log.logBasic(
+          "Built fat jar "
+              + jar
+              + " in "
+              + (System.currentTimeMillis() - start) / 1000
+              + "s (sha256="
+              + sha256Of(jar)
+              + ")");
+      removeStale(log, folder, flavor, jar);
+      return jar;
+    }
+  }
+
+  /**
+   * SHA-256 of a jar's contents: read from the {@code .sha256} sidecar when 
present, computed
+   * otherwise.
+   */
+  public static String sha256Of(Path jar) throws HopException {
+    Path sidecar = sidecar(jar);
+    if (Files.isRegularFile(sidecar)) {
+      try {
+        String sha = Files.readString(sidecar, StandardCharsets.UTF_8).trim();
+        if (sha.length() == 64) {
+          return sha.toLowerCase();
+        }
+      } catch (IOException e) {
+        // compute below
+      }
+    }
+    return sha256Hex(jar);
+  }
+
+  /** SHA-256 hex (lowercase) of a file. */
+  public static String sha256Hex(Path file) throws HopException {
+    try {
+      MessageDigest digest = MessageDigest.getInstance("SHA-256");
+      try (InputStream in = Files.newInputStream(file);
+          DigestInputStream din = new DigestInputStream(in, digest)) {
+        din.transferTo(OutputStream.nullOutputStream());
+      }
+      return HexFormat.of().formatHex(digest.digest());
+    } catch (Exception e) {
+      throw new HopException("Unable to compute SHA-256 of " + file, e);
+    }
+  }
+
+  static Path cacheFolder(IVariables variables) {
+    String folder = variables != null ? 
variables.getVariable(VARIABLE_CACHE_FOLDER) : null;
+    if (StringUtils.isBlank(folder)) {
+      folder = System.getProperty(VARIABLE_CACHE_FOLDER);
+    }
+    if (StringUtils.isBlank(folder)) {
+      folder = System.getenv(VARIABLE_CACHE_FOLDER);
+    }
+    if (StringUtils.isBlank(folder)) {
+      return Paths.get(System.getProperty("java.io.tmpdir"), "hop-fat-jars");

Review Comment:
   **Should fix:** the cache defaults to `java.io.tmpdir/hop-fat-jars`, and 
`hop-conf --fat-jar-engine=spark` now always goes through it. On a shared 
machine such as an edge node, the first user creates `/tmp/hop-fat-jars` with 
mode 755, so another user's build can't write its temp file there. Where `/tmp` 
is a tmpfs, each cached jar (~465 MB) also sits in memory for up to 7 days. 
Suggested fix: a per-user default folder, for example under the user's home or 
the Hop config folder.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to