This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
The following commit(s) were added to refs/heads/master by this push:
new b3f10318a [common] Load plugin classes directly from jar without
decompression (#1996)
b3f10318a is described below
commit b3f10318a0219d4de78303854fb252bfe0a274f6
Author: tsreaper <[email protected]>
AuthorDate: Fri Sep 15 17:58:22 2023 +0800
[common] Load plugin classes directly from jar without decompression (#1996)
---
paimon-codegen-loader/pom.xml | 7 +-
.../org/apache/paimon/codegen/CodeGenLoader.java | 4 +-
.../org/apache/paimon/plugin/PluginLoader.java | 86 +++-------------------
paimon-filesystems/paimon-oss/pom.xml | 7 +-
.../main/java/org/apache/paimon/oss/OSSLoader.java | 4 +-
paimon-filesystems/paimon-s3/pom.xml | 7 +-
.../main/java/org/apache/paimon/s3/S3Loader.java | 4 +-
pom.xml | 2 +-
8 files changed, 28 insertions(+), 93 deletions(-)
diff --git a/paimon-codegen-loader/pom.xml b/paimon-codegen-loader/pom.xml
index 1cbb1ea00..addd8cea8 100644
--- a/paimon-codegen-loader/pom.xml
+++ b/paimon-codegen-loader/pom.xml
@@ -65,10 +65,10 @@ under the License.
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
- <id>copy-table-planner-jars</id>
+ <id>copy-codegen-classes</id>
<phase>prepare-package</phase>
<goals>
- <goal>copy</goal>
+ <goal>unpack</goal>
</goals>
<configuration>
<artifactItems>
@@ -78,10 +78,9 @@ under the License.
<version>${project.version}</version>
<type>jar</type>
<overWrite>true</overWrite>
-
<destFileName>paimon-codegen.jar</destFileName>
</artifactItem>
</artifactItems>
-
<outputDirectory>${project.build.directory}/classes</outputDirectory>
+
<outputDirectory>${project.build.directory}/classes/paimon-codegen</outputDirectory>
</configuration>
</execution>
</executions>
diff --git
a/paimon-codegen-loader/src/main/java/org/apache/paimon/codegen/CodeGenLoader.java
b/paimon-codegen-loader/src/main/java/org/apache/paimon/codegen/CodeGenLoader.java
index 3dbf4db36..df38cae8a 100644
---
a/paimon-codegen-loader/src/main/java/org/apache/paimon/codegen/CodeGenLoader.java
+++
b/paimon-codegen-loader/src/main/java/org/apache/paimon/codegen/CodeGenLoader.java
@@ -23,7 +23,7 @@ import org.apache.paimon.plugin.PluginLoader;
/** Loader to load codegen classes. */
public class CodeGenLoader {
- private static final String CODEGEN_FAT_JAR = "paimon-codegen.jar";
+ private static final String CODEGEN_CLASSES_DIR = "paimon-codegen";
// Singleton lazy initialization
@@ -32,7 +32,7 @@ public class CodeGenLoader {
private static synchronized PluginLoader getLoader() {
if (loader == null) {
// Avoid NoClassDefFoundError without cause by exception
- loader = new PluginLoader(CODEGEN_FAT_JAR);
+ loader = new PluginLoader(CODEGEN_CLASSES_DIR);
}
return loader;
}
diff --git
a/paimon-common/src/main/java/org/apache/paimon/plugin/PluginLoader.java
b/paimon-common/src/main/java/org/apache/paimon/plugin/PluginLoader.java
index 019b7e25c..52100e156 100644
--- a/paimon-common/src/main/java/org/apache/paimon/plugin/PluginLoader.java
+++ b/paimon-common/src/main/java/org/apache/paimon/plugin/PluginLoader.java
@@ -18,24 +18,11 @@
package org.apache.paimon.plugin;
-import org.apache.paimon.utils.FileIOUtils;
-import org.apache.paimon.utils.IOUtils;
-import org.apache.paimon.utils.LocalFileUtils;
-import org.apache.paimon.utils.StringUtils;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
import java.net.URL;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-import java.util.Optional;
import java.util.ServiceLoader;
-import java.util.UUID;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -62,54 +49,22 @@ public class PluginLoader {
private static final String[] COMPONENT_CLASSPATH = new String[]
{"org.apache.paimon"};
- private final Path delegateJar;
-
private final ComponentClassLoader submoduleClassLoader;
- public PluginLoader(String jarName) {
- try {
- ClassLoader ownerClassLoader = PluginLoader.class.getClassLoader();
- Path tmpDirectory =
- tmpDirectoryFromYarn()
- .orElseGet(() ->
Paths.get(System.getProperty("java.io.tmpdir")));
- Files.createDirectories(
-
LocalFileUtils.getTargetPathIfContainsSymbolicPath(tmpDirectory));
- delegateJar =
- extractResource(
- jarName,
- ownerClassLoader,
- tmpDirectory,
- String.format(
- "%s could not be found.\n"
- + "If you're running a test,
please make sure you've built the modules by running\n"
- + "mvn clean install -DskipTests",
- jarName));
- this.submoduleClassLoader =
- new ComponentClassLoader(
- new URL[] {delegateJar.toUri().toURL()},
- ownerClassLoader,
- OWNER_CLASSPATH,
- COMPONENT_CLASSPATH);
- } catch (IOException e) {
- throw new RuntimeException("Could not initialize the
paimon-codegen loader.", e);
+ public PluginLoader(String dirName) {
+ // URL must end with slash,
+ // otherwise URLClassLoader cannot recognize this path as a directory
+ if (!dirName.endsWith("/")) {
+ dirName += "/";
}
- }
- private Path extractResource(
- String resourceName,
- ClassLoader flinkClassLoader,
- Path tmpDirectory,
- String errorMessage)
- throws IOException {
- String[] splitName = resourceName.split("\\.");
- splitName[0] += "_" + UUID.randomUUID();
- final Path tempFile =
Files.createFile(tmpDirectory.resolve(String.join(".", splitName)));
- final InputStream resourceStream =
flinkClassLoader.getResourceAsStream(resourceName);
- if (resourceStream == null) {
- throw new RuntimeException(errorMessage);
- }
- IOUtils.copyBytes(resourceStream, Files.newOutputStream(tempFile));
- return tempFile;
+ ClassLoader ownerClassLoader = PluginLoader.class.getClassLoader();
+ this.submoduleClassLoader =
+ new ComponentClassLoader(
+ new URL[] {ownerClassLoader.getResource(dirName)},
+ ownerClassLoader,
+ OWNER_CLASSPATH,
+ COMPONENT_CLASSPATH);
}
public <T> T discover(Class<T> clazz) {
@@ -141,21 +96,4 @@ public class PluginLoader {
public ClassLoader submoduleClassLoader() {
return submoduleClassLoader;
}
-
- @Override
- protected void finalize() throws IOException {
- submoduleClassLoader.close();
- FileIOUtils.deleteFileOrDirectory(delegateJar.toFile());
- }
-
- private static Optional<Path> tmpDirectoryFromYarn() {
- String localDirs = System.getenv("LOCAL_DIRS");
- if (!StringUtils.isBlank(localDirs)) {
- String[] paths = localDirs.split(",|" + File.pathSeparator);
- if (paths.length > 0) {
- return Optional.of(Paths.get(paths[0]));
- }
- }
- return Optional.empty();
- }
}
diff --git a/paimon-filesystems/paimon-oss/pom.xml
b/paimon-filesystems/paimon-oss/pom.xml
index d0bc4f9a7..d42a0ea7f 100644
--- a/paimon-filesystems/paimon-oss/pom.xml
+++ b/paimon-filesystems/paimon-oss/pom.xml
@@ -61,10 +61,10 @@
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
- <id>copy-oss-jar</id>
+ <id>copy-oss-classes</id>
<phase>prepare-package</phase>
<goals>
- <goal>copy</goal>
+ <goal>unpack</goal>
</goals>
<configuration>
<artifactItems>
@@ -74,10 +74,9 @@
<version>${project.version}</version>
<type>jar</type>
<overWrite>true</overWrite>
-
<destFileName>paimon-plugin-oss.jar</destFileName>
+
<outputDirectory>${project.build.directory}/classes/paimon-plugin-oss</outputDirectory>
</artifactItem>
</artifactItems>
-
<outputDirectory>${project.build.directory}/classes</outputDirectory>
</configuration>
</execution>
</executions>
diff --git
a/paimon-filesystems/paimon-oss/src/main/java/org/apache/paimon/oss/OSSLoader.java
b/paimon-filesystems/paimon-oss/src/main/java/org/apache/paimon/oss/OSSLoader.java
index 01540239d..8311b8014 100644
---
a/paimon-filesystems/paimon-oss/src/main/java/org/apache/paimon/oss/OSSLoader.java
+++
b/paimon-filesystems/paimon-oss/src/main/java/org/apache/paimon/oss/OSSLoader.java
@@ -31,7 +31,7 @@ import java.util.List;
/** A {@link PluginLoader} to load oss. */
public class OSSLoader implements FileIOLoader {
- private static final String OSS_JAR = "paimon-plugin-oss.jar";
+ private static final String OSS_CLASSES_DIR = "paimon-plugin-oss";
private static final String OSS_CLASS = "org.apache.paimon.oss.OSSFileIO";
@@ -42,7 +42,7 @@ public class OSSLoader implements FileIOLoader {
private static synchronized PluginLoader getLoader() {
if (loader == null) {
// Avoid NoClassDefFoundError without cause by exception
- loader = new PluginLoader(OSS_JAR);
+ loader = new PluginLoader(OSS_CLASSES_DIR);
}
return loader;
}
diff --git a/paimon-filesystems/paimon-s3/pom.xml
b/paimon-filesystems/paimon-s3/pom.xml
index 5e44a4ae3..7213572c8 100644
--- a/paimon-filesystems/paimon-s3/pom.xml
+++ b/paimon-filesystems/paimon-s3/pom.xml
@@ -99,10 +99,10 @@
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
- <id>copy-s3-jar</id>
+ <id>copy-s3-classes</id>
<phase>prepare-package</phase>
<goals>
- <goal>copy</goal>
+ <goal>unpack</goal>
</goals>
<configuration>
<artifactItems>
@@ -112,10 +112,9 @@
<version>${project.version}</version>
<type>jar</type>
<overWrite>true</overWrite>
-
<destFileName>paimon-plugin-s3.jar</destFileName>
+
<outputDirectory>${project.build.directory}/classes/paimon-plugin-s3</outputDirectory>
</artifactItem>
</artifactItems>
-
<outputDirectory>${project.build.directory}/classes</outputDirectory>
</configuration>
</execution>
</executions>
diff --git
a/paimon-filesystems/paimon-s3/src/main/java/org/apache/paimon/s3/S3Loader.java
b/paimon-filesystems/paimon-s3/src/main/java/org/apache/paimon/s3/S3Loader.java
index 1f1b66a86..02ffd7b4c 100644
---
a/paimon-filesystems/paimon-s3/src/main/java/org/apache/paimon/s3/S3Loader.java
+++
b/paimon-filesystems/paimon-s3/src/main/java/org/apache/paimon/s3/S3Loader.java
@@ -31,7 +31,7 @@ import java.util.List;
/** A {@link PluginLoader} to load oss. */
public class S3Loader implements FileIOLoader {
- private static final String S3_JAR = "paimon-plugin-s3.jar";
+ private static final String S3_CLASSES_DIR = "paimon-plugin-s3";
private static final String S3_CLASS = "org.apache.paimon.s3.S3FileIO";
@@ -42,7 +42,7 @@ public class S3Loader implements FileIOLoader {
private static synchronized PluginLoader getLoader() {
if (loader == null) {
// Avoid NoClassDefFoundError without cause by exception
- loader = new PluginLoader(S3_JAR);
+ loader = new PluginLoader(S3_CLASSES_DIR);
}
return loader;
}
diff --git a/pom.xml b/pom.xml
index bb3c4cda6..474220096 100644
--- a/pom.xml
+++ b/pom.xml
@@ -70,7 +70,7 @@ under the License.
<properties>
<project.build.outputTimestamp>2023-09-06T02:16:30Z</project.build.outputTimestamp>
- <paimon.shade.version>0.4.0-incubating</paimon.shade.version>
+ <paimon.shade.version>0.6-SNAPSHOT</paimon.shade.version>
<paimon.shade.jackson.version>2.14.2</paimon.shade.jackson.version>
<paimon.shade.guava.version>30.1.1-jre</paimon.shade.guava.version>
<paimon.shade.caffeine.version>2.9.3</paimon.shade.caffeine.version>
