bamaer commented on code in PR #9595: URL: https://github.com/apache/hop/pull/9595#discussion_r4653218664
########## core/src/main/java/org/apache/hop/core/fatjar/NativeSparkFatJar.java: ########## @@ -0,0 +1,303 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.core.fatjar; + +import java.io.File; +import java.nio.file.Path; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.apache.hop.core.IProgressMonitor; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.logging.ILogChannel; +import org.apache.hop.core.variables.IVariables; + +/** + * Which installed jars go into a fat jar for the native Spark engine (Spark 4, Scala 2.13). + * + * <p>Two flavors: + * + * <ul> + * <li><b>native</b>: leaves out Beam's Spark 3 / Scala 2.12 client classpath and keeps the Spark + * 4 runtime of {@code plugins/engines/spark/lib}. + * <li><b>native-provided</b>: also leaves out every Spark, Scala and Hadoop client runtime jar, + * because the cluster provides them. This is the flavor for a real cluster: standalone, YARN + * (where {@code spark.yarn.jars} ships the engine's Spark runtime), Kubernetes and + * Databricks. + * </ul> + * + * <p>Shared by the Spark engine (automatic fat jar), the Databricks deployment and {@code hop-conf + * --generate-fat-jar --fat-jar-engine=spark}. + */ +public final class NativeSparkFatJar { + + /** + * Written by the engine plugin at build time: its Maven dependency tree, which tells what a Spark + * distribution of the engine's Spark version ships. + */ + public static final String SPARK_DEPENDENCIES_FILE = + "plugins/engines/spark/spark-dependencies.txt"; + + /** Cache flavor of a native-provided fat jar: {@code hop-native-provided-<fingerprint>.jar}. */ + public static final String FLAVOR_PROVIDED = "hop-native-provided"; + + /** Cache flavor of a native fat jar that embeds the Spark runtime. */ + public static final String FLAVOR_NATIVE = "hop-native"; + + private NativeSparkFatJar() {} + + /** + * @param provided true to leave out the Spark/Scala/Hadoop runtime the cluster provides + * @return the jar filenames of the installation in the working directory + */ + public static List<String> findJarFilenames(boolean provided) { + return findJarFilenames(new File("."), provided); + } + + /** + * @param installation the Hop installation folder + * @param provided true to leave out what the Spark cluster provides + */ + public static List<String> findJarFilenames(File installation, boolean provided) { + Set<String> sparkArtifacts = provided ? sparkDistributionArtifacts(installation) : Set.of(); + return InstalledJars.find( + installation, + file -> + isExcluded(file, provided) + || (!file.isDirectory() && isSparkDistributionJar(file, sparkArtifacts))); + } + + /** + * The artifact ids a Spark distribution of the engine's Spark version ships, from the dependency + * tree the engine plugin writes at build time ({@value #SPARK_DEPENDENCIES_FILE}): everything + * under its {@code org.apache.spark} and {@code org.apache.hadoop} dependencies. Empty when the + * file is missing, for an engine plugin built before it existed. + */ + static Set<String> sparkDistributionArtifacts(File installation) { + File tree = new File(installation, SPARK_DEPENDENCIES_FILE); + if (!tree.isFile()) { + return Set.of(); + } + try { + return parseDistributionArtifacts( + java.nio.file.Files.readAllLines(tree.toPath(), java.nio.charset.StandardCharsets.UTF_8)); + } catch (java.io.IOException e) { + return Set.of(); + } + } + + /** + * Parse {@code mvn dependency:tree} text output: the artifact ids of the top-level {@code + * org.apache.spark} and {@code org.apache.hadoop} dependencies and everything below them. + */ + static Set<String> parseDistributionArtifacts(List<String> lines) { + Set<String> artifacts = new HashSet<>(); + boolean inDistribution = false; + for (String line : lines) { + int start = 0; + while (start < line.length() && !Character.isLetterOrDigit(line.charAt(start))) { + start++; + } + if (start == 0 || start >= line.length()) { + continue; // the root (the engine plugin itself) or an empty line + } + String[] coordinates = line.substring(start).trim().split(":"); + if (coordinates.length < 4) { + continue; + } + if (start == 3) { + // a direct dependency of the engine plugin: does a Spark distribution ship it? + inDistribution = + "org.apache.spark".equals(coordinates[0]) || "org.apache.hadoop".equals(coordinates[0]); + } + if (inDistribution) { + artifacts.add(coordinates[1]); + } + } + return artifacts; + } + + /** + * A jar named after one of these artifacts: {@code <artifactId>-<version>[-classifier].jar}. The + * lakehouse connectors in the engine's lib/delta and lib/iceberg always stay: no Spark + * distribution ships them. + */ + static boolean isSparkDistributionJar(File jar, Set<String> sparkArtifacts) { Review Comment: **Blocker:** the provided jar now leaves out jars that the cluster doesn't provide. Jars are matched on artifactId only, against the Maven tree: - `plugins/transforms/script/lib/xz-25.0.4.jar` and `icu4j-25.0.4.jar` are GraalVM's shadowed copies (`org/graalvm/shadowed/...`). They are dropped because Spark ships `org.tukaani:xz` and `com.ibm.icu:icu4j`. - `lib/core/protobuf-java-4.34.1.jar` is dropped, but Spark 4.1.3 shades protobuf: its `jars/` folder has no `com/google/protobuf` classes. The gs:// file system (`google-cloud-storage`) and the Google Secret Manager resolver use them. Generate rows → Script (JavaScript) → Text file output on the cluster fails on every executor with this jar: ``` NoClassDefFoundError: org/graalvm/shadowed/com/ibm/icu/text/DateFormat ``` The same install with main's jar selection writes the 100 rows. Suggested fix: apply the dependency-tree exclusion to `plugins/engines/spark/lib` only. Those are the Spark runtime copies, and that's most of the saving (153 of the ~204 MB it removes, uncompressed). A unit test with a shadowed `xz` in a transform plugin and `protobuf-java` in `lib/core` would cover it. ########## core/src/main/java/org/apache/hop/core/fatjar/FatJarCache.java: ########## @@ -0,0 +1,266 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.core.fatjar; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.nio.file.attribute.FileTime; +import java.security.DigestInputStream; +import java.security.MessageDigest; +import java.time.Duration; +import java.time.Instant; +import java.util.HexFormat; +import java.util.List; +import java.util.UUID; +import org.apache.commons.lang3.StringUtils; +import org.apache.hop.core.IProgressMonitor; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.logging.ILogChannel; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.i18n.BaseMessages; + +/** + * A local cache of fat jars, so a run does not rebuild a fat jar the installation already has. + * + * <p>A cached jar is named after its flavor and a fingerprint of its input jars (path, size and + * modification time): {@code <flavor>-<fingerprint>.jar}. Installing or upgrading a plugin changes + * the fingerprint, and the next run builds a new jar. Next to the jar a {@code .sha256} sidecar + * holds the SHA-256 of the jar contents, so a deployment can name and compare the jar it uploads + * without hashing a few hundred megabytes again. + * + * <p>The cache folder is {@value #VARIABLE_CACHE_FOLDER} (variable, system property or environment + * variable), or {@code hop-fat-jars} in the temporary folder. Jars of the same flavor that were not + * used for {@link #RETENTION} are removed after a build. + */ +public final class FatJarCache { + + /** Folder of the fat jar cache. */ + public static final String VARIABLE_CACHE_FOLDER = "HOP_FAT_JAR_CACHE_FOLDER"; + + public static final String SHA256_SIDECAR_SUFFIX = ".sha256"; + + /** Number of fingerprint hex characters in a cached jar name. */ + static final int FINGERPRINT_LENGTH = 16; + + /** Unused jars of a flavor older than this are removed after a new build. */ + static final Duration RETENTION = Duration.ofDays(7); + + private static final Object BUILD_LOCK = new Object(); + + private static final Class<?> PKG = FatJarCache.class; + + private FatJarCache() {} + + /** + * @param log log channel for the build + * @param variables resolves {@value #VARIABLE_CACHE_FOLDER} + * @param flavor jar name prefix, for example {@link NativeSparkFatJar#FLAVOR_PROVIDED} + * @param jarFilenames the jars to merge + * @return the cached fat jar for exactly these input jars + */ + public static Path getOrBuild( + ILogChannel log, IVariables variables, String flavor, List<String> jarFilenames) + throws HopException { + return getOrBuild(log, variables, flavor, jarFilenames, null); + } + + /** + * @param progressMonitor optional: follows a build and can cancel it; a cached jar is returned + * without using it + * @see #getOrBuild(ILogChannel, IVariables, String, List) + */ + public static Path getOrBuild( + ILogChannel log, + IVariables variables, + String flavor, + List<String> jarFilenames, + IProgressMonitor progressMonitor) + throws HopException { + Path folder = cacheFolder(variables); + String fingerprint = fingerprint(flavor, jarFilenames); + Path jar = folder.resolve(flavor + "-" + fingerprint + ".jar"); + + synchronized (BUILD_LOCK) { + if (Files.isRegularFile(jar) && Files.isRegularFile(sidecar(jar))) { + touch(jar); + log.logBasic("Using cached fat jar " + jar); + return jar; + } + + long start = System.currentTimeMillis(); + log.logBasic( + "Building fat jar " + + jar + + " from " + + jarFilenames.size() + + " jar files. This happens once per installation change."); + try { + Files.createDirectories(folder); + } catch (IOException e) { + throw new HopException("Unable to create the fat jar cache folder " + folder, e); + } + // Build next to the final name and move it in place, so a reader never sees half a jar. + Path temp = folder.resolve(jar.getFileName() + ".tmp-" + UUID.randomUUID()); + try { + FatJarBuilder builder = new FatJarBuilder(log, variables, temp.toString(), jarFilenames); + builder.setProgressMonitor(progressMonitor); + builder.buildTargetJar(); + if (progressMonitor != null) { + progressMonitor.subTask(BaseMessages.getString(PKG, "FatJarCache.Progress.Checksum")); + } + String sha = sha256Hex(temp); + Path tempSidecar = folder.resolve(temp.getFileName() + SHA256_SIDECAR_SUFFIX); + Files.writeString(tempSidecar, sha + "\n", StandardCharsets.UTF_8); + Files.move(temp, jar, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); + Files.move( + tempSidecar, + sidecar(jar), + StandardCopyOption.REPLACE_EXISTING, + StandardCopyOption.ATOMIC_MOVE); + } catch (IOException e) { + throw new HopException("Unable to store fat jar " + jar, e); + } finally { + deleteQuietly(temp); + deleteQuietly(folder.resolve(temp.getFileName() + SHA256_SIDECAR_SUFFIX)); + } + log.logBasic( + "Built fat jar " + + jar + + " in " + + (System.currentTimeMillis() - start) / 1000 + + "s (sha256=" + + sha256Of(jar) + + ")"); + removeStale(log, folder, flavor, jar); + return jar; + } + } + + /** + * SHA-256 of a jar's contents: read from the {@code .sha256} sidecar when present, computed + * otherwise. + */ + public static String sha256Of(Path jar) throws HopException { + Path sidecar = sidecar(jar); + if (Files.isRegularFile(sidecar)) { + try { + String sha = Files.readString(sidecar, StandardCharsets.UTF_8).trim(); + if (sha.length() == 64) { + return sha.toLowerCase(); + } + } catch (IOException e) { + // compute below + } + } + return sha256Hex(jar); + } + + /** SHA-256 hex (lowercase) of a file. */ + public static String sha256Hex(Path file) throws HopException { + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + try (InputStream in = Files.newInputStream(file); + DigestInputStream din = new DigestInputStream(in, digest)) { + din.transferTo(OutputStream.nullOutputStream()); + } + return HexFormat.of().formatHex(digest.digest()); + } catch (Exception e) { + throw new HopException("Unable to compute SHA-256 of " + file, e); + } + } + + static Path cacheFolder(IVariables variables) { + String folder = variables != null ? variables.getVariable(VARIABLE_CACHE_FOLDER) : null; + if (StringUtils.isBlank(folder)) { + folder = System.getProperty(VARIABLE_CACHE_FOLDER); + } + if (StringUtils.isBlank(folder)) { + folder = System.getenv(VARIABLE_CACHE_FOLDER); + } + if (StringUtils.isBlank(folder)) { + return Paths.get(System.getProperty("java.io.tmpdir"), "hop-fat-jars"); Review Comment: **Should fix:** the cache defaults to `java.io.tmpdir/hop-fat-jars`, and `hop-conf --fat-jar-engine=spark` now always goes through it. On a shared machine such as an edge node, the first user creates `/tmp/hop-fat-jars` with mode 755, so another user's build can't write its temp file there. Where `/tmp` is a tmpfs, each cached jar (~465 MB) also sits in memory for up to 7 days. Suggested fix: a per-user default folder, for example under the user's home or the Hop config folder. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
