This is an automated email from the ASF dual-hosted git repository.
lidavidm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 3217fc2625 GH-44564: [Java][FlightSQL] Fix native libraries relocation
(#44565)
3217fc2625 is described below
commit 3217fc2625f80042661172fd589a70c270ba8f8b
Author: Laurent Goujon <lauren...@users.noreply.github.com>
AuthorDate: Tue Oct 29 21:15:50 2024 -0700
GH-44564: [Java][FlightSQL] Fix native libraries relocation (#44565)
### Rationale for this change
Prefix used by native libraries shipped in the JDBC driver do not match the
prefix used by Netty `NativeLibraryLoader` class, preventing them to be
detected and loaded.
### What changes are included in this PR?
Change the prefix of the libraries and add a integration test to verify the
libraries are loaded
Also exclude several group of data which are not properly relocated and may
cause conflict with existing classpath.
### Are these changes tested?
Yes, new test added for checking the native library loader
### Are there any user-facing changes?
No
* GitHub Issue: #44564
Authored-by: Laurent Goujon <laur...@apache.org>
Signed-off-by: David Li <li.david...@gmail.com>
---
java/flight/flight-sql-jdbc-driver/pom.xml | 13 +--
.../arrow/driver/jdbc/ITDriverJarValidation.java | 96 ++++++++++++++++------
2 files changed, 79 insertions(+), 30 deletions(-)
diff --git a/java/flight/flight-sql-jdbc-driver/pom.xml
b/java/flight/flight-sql-jdbc-driver/pom.xml
index ba6fe277d1..ae8c543fbf 100644
--- a/java/flight/flight-sql-jdbc-driver/pom.xml
+++ b/java/flight/flight-sql-jdbc-driver/pom.xml
@@ -127,14 +127,14 @@ under the License.
<pattern>mozilla.</pattern>
<shadedPattern>org.apache.arrow.driver.jdbc.shaded.mozilla.</shadedPattern>
</relocation>
- <!-- Entries to relocate netty native libraries -->
+ <!-- Entries to relocate netty native libraries. Prefix has to
match relocation prefix (dots replaced with underscore) -->
<relocation>
<pattern>META-INF.native.libnetty_</pattern>
-
<shadedPattern>META-INF.native.liboaadj_netty_</shadedPattern>
+
<shadedPattern>META-INF.native.liborg_apache_arrow_driver_jdbc_shaded_netty_</shadedPattern>
</relocation>
<relocation>
<pattern>META-INF.native.netty_</pattern>
- <shadedPattern>META-INF.native.oaadj_netty_</shadedPattern>
+
<shadedPattern>META-INF.native.org_apache_arrow_driver_jdbc_shaded_netty_</shadedPattern>
</relocation>
</relocations>
<transformers>
@@ -159,8 +159,11 @@ under the License.
<exclude>**/*.SF</exclude>
<exclude>**/*.RSA</exclude>
<exclude>**/*.DSA</exclude>
- <exclude>META-INF/native/libio_grpc_netty*</exclude>
- <exclude>META-INF/native/io_grpc_netty_shaded*</exclude>
+ <!-- Requires some resource transformer -->
+ <exclude>META-INF/native-image/</exclude>
+ <exclude>META-INF/proguard/</exclude>
+ <!-- Requires MSHADE-406 -->
+ <exclude>META-INF/versions/</exclude>
<exclude>**/*.proto</exclude>
<exclude>**/module-info.class</exclude>
</excludes>
diff --git
a/java/flight/flight-sql-jdbc-driver/src/test/java/org/apache/arrow/driver/jdbc/ITDriverJarValidation.java
b/java/flight/flight-sql-jdbc-driver/src/test/java/org/apache/arrow/driver/jdbc/ITDriverJarValidation.java
index b45845485a..a0e108d6a0 100644
---
a/java/flight/flight-sql-jdbc-driver/src/test/java/org/apache/arrow/driver/jdbc/ITDriverJarValidation.java
+++
b/java/flight/flight-sql-jdbc-driver/src/test/java/org/apache/arrow/driver/jdbc/ITDriverJarValidation.java
@@ -18,20 +18,26 @@ package org.apache.arrow.driver.jdbc;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.fail;
import com.google.common.collect.ImmutableSet;
import java.io.File;
import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
import java.net.JarURLConnection;
+import java.net.URISyntaxException;
import java.net.URL;
-import java.util.Enumeration;
+import java.net.URLClassLoader;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import java.util.function.Predicate;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
+import java.util.stream.Stream;
+import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.api.function.Executable;
/**
* Check the content of the JDBC driver jar
@@ -48,52 +54,92 @@ public class ITDriverJarValidation {
/** List of allowed prefixes a jar entry may match. */
public static final Set<String> ALLOWED_PREFIXES =
- ImmutableSet.of("org/apache/arrow/driver/jdbc/", "META-INF/");
+ ImmutableSet.of(
+ "org/apache/arrow/driver/jdbc/", // Driver code
+ "META-INF/maven/", // Maven metadata (useful for security scanner
+ "META-INF/services/", // ServiceLoader implementations
+ "META-INF/license/",
+ "META-INF/licenses/",
+ // Prefixes for native libraries
+ "META-INF/native/liborg_apache_arrow_driver_jdbc_shaded_",
+ "META-INF/native/org_apache_arrow_driver_jdbc_shaded_");
/** List of allowed files a jar entry may match. */
public static final Set<String> ALLOWED_FILES =
- ImmutableSet.of("arrow-git.properties", "properties/flight.properties");
+ ImmutableSet.of(
+ "arrow-git.properties",
+ "properties/flight.properties",
+ "META-INF/io.netty.versions.properties",
+ "META-INF/MANIFEST.MF",
+ "META-INF/DEPENDENCIES",
+ "META-INF/FastDoubleParser-LICENSE",
+ "META-INF/FastDoubleParser-NOTICE",
+ "META-INF/LICENSE",
+ "META-INF/LICENSE.txt",
+ "META-INF/NOTICE",
+ "META-INF/NOTICE.txt",
+ "META-INF/thirdparty-LICENSE",
+ "META-INF/bigint-LICENSE");
// This method is designed to work with Maven failsafe plugin and expects the
// JDBC driver jar to be present in the test classpath (instead of the
individual classes)
- private static JarFile getJdbcJarFile() throws IOException {
+ private static File getJdbcJarFile() throws IOException {
// Check if an override has been set
if (JDBC_DRIVER_PATH_OVERRIDE != null) {
- return new JarFile(new File(JDBC_DRIVER_PATH_OVERRIDE));
+ return new File(JDBC_DRIVER_PATH_OVERRIDE);
}
- // Check classpath to find the driver jar
+ // Check classpath to find the driver jar (without loading the class)
URL driverClassURL =
ITDriverJarValidation.class
.getClassLoader()
.getResource("org/apache/arrow/driver/jdbc/ArrowFlightJdbcDriver.class");
- assertNotNull(driverClassURL, "Driver jar was not detected in the
classpath");
+ assertNotNull(driverClassURL, "Driver class was not detected in the
classpath");
assertEquals(
- "jar", driverClassURL.getProtocol(), "Driver jar was not detected in
the classpath");
+ "jar", driverClassURL.getProtocol(), "Driver class was not found
inside a jar file");
+ // Return the enclosing jar file
JarURLConnection connection = (JarURLConnection)
driverClassURL.openConnection();
- return connection.getJarFile();
+ try {
+ return new File(connection.getJarFileURL().toURI());
+ } catch (URISyntaxException e) {
+ throw new IOException(e);
+ }
}
+ /** Validate the content of the jar to enforce all 3rd party dependencies
have been shaded. */
@Test
@Timeout(value = 2, unit = TimeUnit.MINUTES)
public void validateShadedJar() throws IOException {
- // Validate the content of the jar to enforce all 3rd party dependencies
have
- // been shaded
- try (JarFile jar = getJdbcJarFile()) {
- for (Enumeration<JarEntry> entries = jar.entries();
entries.hasMoreElements(); ) {
- final JarEntry entry = entries.nextElement();
- if (entry.isDirectory()) {
- // Directories are ignored
- continue;
- }
-
- try {
- checkEntryAllowed(entry.getName());
- } catch (AssertionError e) {
- fail(e.getMessage());
- }
+
+ try (JarFile jar = new JarFile(getJdbcJarFile())) {
+ Stream<Executable> executables =
+ jar.stream()
+ .filter(Predicate.not(JarEntry::isDirectory))
+ .map(
+ entry -> {
+ return () -> checkEntryAllowed(entry.getName());
+ });
+
+ Assertions.assertAll(executables);
+ }
+ }
+
+ /** Check that relocated netty code can also load matching native library. */
+ @Test
+ @Timeout(value = 2, unit = TimeUnit.MINUTES)
+ public void checkNettyOpenSslNativeLoader() throws Throwable {
+ try (URLClassLoader driverClassLoader =
+ new URLClassLoader(new URL[] {getJdbcJarFile().toURI().toURL()},
null)) {
+ Class<?> openSslClass =
+ driverClassLoader.loadClass(
+
"org.apache.arrow.driver.jdbc.shaded.io.netty.handler.ssl.OpenSsl");
+ Method method = openSslClass.getDeclaredMethod("ensureAvailability");
+ try {
+ method.invoke(null);
+ } catch (InvocationTargetException e) {
+ throw e.getCause();
}
}
}
