This is an automated email from the ASF dual-hosted git repository. bonampak pushed a commit to branch feature/jakarta-jetty-upgrade in repository https://gitbox.apache.org/repos/asf/knox.git
commit 29d6df8b377a747166b5bca6251ad43f70c9dee1 Author: bonampak <[email protected]> AuthorDate: Wed Jul 22 11:51:29 2026 +0200 KNOX-3238: KNOX-3309: Fix Knox HadoopAuthFilter dependency on hadoop AuthenticationFilter. --- .../main/resources/build-tools/spotbugs-filter.xml | 8 + .../pom.xml | 5 - gateway-provider-security-hadoopauth/pom.xml | 14 +- .../hadoopauth/filter/HadoopAuthFilter.java | 14 +- .../hadoopauth/filter/HadoopAuthFilterTest.java | 3 - gateway-server/pom.xml | 9 +- gateway-shim-hadoop-auth/pom.xml | 149 +++++++++++++++++++ .../gateway/shim/hadoopauth/Servlet6Patcher.java | 163 +++++++++++++++++++++ gateway-topology-hadoop-xml/pom.xml | 1 + pom.xml | 26 ++++ 10 files changed, 366 insertions(+), 26 deletions(-) diff --git a/build-tools/src/main/resources/build-tools/spotbugs-filter.xml b/build-tools/src/main/resources/build-tools/spotbugs-filter.xml index fb4d7857d..f9173e595 100644 --- a/build-tools/src/main/resources/build-tools/spotbugs-filter.xml +++ b/build-tools/src/main/resources/build-tools/spotbugs-filter.xml @@ -25,6 +25,14 @@ limitations under the License. <Bug pattern="PATH_TRAVERSAL_IN" /> </Match> + <!-- Servlet6Patcher is a build-time utility invoked with a fixed argument + from the shim module's pom (target/<finalName>.jar). It only ever runs + during the reactor build. --> + <Match> + <Class name="org.apache.knox.gateway.shim.hadoopauth.Servlet6Patcher" /> + <Bug pattern="PATH_TRAVERSAL_IN" /> + </Match> + <Match> <Class name="org.apache.knox.gateway.util.X509CertificateUtil" /> <Method name="writeCertificateToKeyStore" /> diff --git a/gateway-provider-identity-assertion-hadoop-groups/pom.xml b/gateway-provider-identity-assertion-hadoop-groups/pom.xml index 12e5dbe32..d0a6876b1 100644 --- a/gateway-provider-identity-assertion-hadoop-groups/pom.xml +++ b/gateway-provider-identity-assertion-hadoop-groups/pom.xml @@ -56,11 +56,6 @@ <artifactId>hadoop-common</artifactId> </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-auth</artifactId> - </dependency> - <dependency> <groupId>jakarta.servlet</groupId> <artifactId>jakarta.servlet-api</artifactId> diff --git a/gateway-provider-security-hadoopauth/pom.xml b/gateway-provider-security-hadoopauth/pom.xml index 76c0908e8..b37200af3 100755 --- a/gateway-provider-security-hadoopauth/pom.xml +++ b/gateway-provider-security-hadoopauth/pom.xml @@ -51,9 +51,12 @@ <artifactId>gateway-util-common</artifactId> </dependency> + <!-- Jakarta-compatible shim of hadoop-auth. Provides + org.apache.hadoop.security.authentication.server.AuthenticationFilter + linked against jakarta.servlet 6 (see gateway-shim-hadoop-auth). --> <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-auth</artifactId> + <groupId>org.apache.knox</groupId> + <artifactId>gateway-shim-hadoop-auth</artifactId> </dependency> <dependency> @@ -66,8 +69,6 @@ <artifactId>jakarta.servlet-api</artifactId> </dependency> - - <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-lang3</artifactId> @@ -77,11 +78,6 @@ <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> </dependency> - <dependency> - <groupId>javax.servlet</groupId> - <artifactId>javax.servlet-api</artifactId> - <version>3.1.0</version> - </dependency> <dependency> <groupId>de.thetaphi</groupId> diff --git a/gateway-provider-security-hadoopauth/src/main/java/org/apache/knox/gateway/hadoopauth/filter/HadoopAuthFilter.java b/gateway-provider-security-hadoopauth/src/main/java/org/apache/knox/gateway/hadoopauth/filter/HadoopAuthFilter.java index 0b6a14b50..0a46b5f62 100755 --- a/gateway-provider-security-hadoopauth/src/main/java/org/apache/knox/gateway/hadoopauth/filter/HadoopAuthFilter.java +++ b/gateway-provider-security-hadoopauth/src/main/java/org/apache/knox/gateway/hadoopauth/filter/HadoopAuthFilter.java @@ -104,7 +104,7 @@ public class HadoopAuthFilter extends private Set<String> unAuthenticatedPaths = new HashSet<>(20); private String topologyName; - //@Override + @Override protected Properties getConfiguration(String configPrefix, FilterConfig filterConfig) throws ServletException { GatewayServices services = GatewayServer.getGatewayServices(); AliasService aliasService = services.getService(ServiceType.ALIAS_SERVICE); @@ -112,7 +112,7 @@ public class HadoopAuthFilter extends return getConfiguration(aliasService, configPrefix, filterConfig); } - //@Override + @Override public void init(FilterConfig filterConfig) throws ServletException { this.topologyName = (String) filterConfig.getInitParameter("clusterName"); final List<String> initParameterNames = AuthFilterUtils.getInitParameterNamesAsList(filterConfig); @@ -144,7 +144,7 @@ public class HadoopAuthFilter extends ignoreDoAs.addAll(ignoredServices); } - //super.init(filterConfig); + super.init(filterConfig); final String supportJwt = filterConfig.getInitParameter(SUPPORT_JWT); final boolean jwtSupported = Boolean.parseBoolean(supportJwt == null ? "false" : supportJwt); @@ -160,7 +160,7 @@ public class HadoopAuthFilter extends AuthFilterUtils.addUnauthPaths(unAuthenticatedPaths, unAuthPathString, DEFAULT_AUTH_UNAUTHENTICATED_PATHS_PARAM); } - //@Override + @Override public void doFilter(ServletRequest request, ServletResponse response, FilterChain filterChain) throws IOException, ServletException { /* check for unauthenticated paths to bypass */ @@ -172,11 +172,11 @@ public class HadoopAuthFilter extends LOG.useJwtFilter(); jwtFilter.doFilter(request, response, filterChain); } else { - //super.doFilter(request, response, filterChain); + super.doFilter(request, response, filterChain); } } - //@Override + @Override protected void doFilter(FilterChain filterChain, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { /* check for unauthenticated paths to bypass */ if(AuthFilterUtils.doesRequestContainUnauthPath(unAuthenticatedPaths, request)) { @@ -218,7 +218,7 @@ public class HadoopAuthFilter extends } } - //super.doFilter(filterChain, proxyRequest == null ? request : proxyRequest, response); + super.doFilter(filterChain, proxyRequest == null ? request : proxyRequest, response); } /** diff --git a/gateway-provider-security-hadoopauth/src/test/java/org/apache/knox/gateway/hadoopauth/filter/HadoopAuthFilterTest.java b/gateway-provider-security-hadoopauth/src/test/java/org/apache/knox/gateway/hadoopauth/filter/HadoopAuthFilterTest.java index 33d91ba15..5c5d39e25 100644 --- a/gateway-provider-security-hadoopauth/src/test/java/org/apache/knox/gateway/hadoopauth/filter/HadoopAuthFilterTest.java +++ b/gateway-provider-security-hadoopauth/src/test/java/org/apache/knox/gateway/hadoopauth/filter/HadoopAuthFilterTest.java @@ -46,7 +46,6 @@ import org.apache.knox.gateway.topology.Topology; import org.easymock.Capture; import org.easymock.EasyMock; import org.junit.Assert; -import org.junit.Ignore; import org.junit.Test; import javax.security.auth.Subject; @@ -65,8 +64,6 @@ import java.util.HashMap; import java.util.Map; import java.util.Properties; -//ignoring until KNOX-3309 fixes HadoopAuthFilter -@Ignore //jetty-12-upgrade-ignore public class HadoopAuthFilterTest { private static final String SERVICE_URL = "https://localhost:8888/gateway/sandbox";; private static final String JWKS_PATH = "/knoxtoken/api/v1/jwks.json"; diff --git a/gateway-server/pom.xml b/gateway-server/pom.xml index c3c3b365c..48d8393b2 100644 --- a/gateway-server/pom.xml +++ b/gateway-server/pom.xml @@ -131,9 +131,14 @@ <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> </dependency> + <!-- Jakarta-compatible shim of hadoop-auth. Only the client-side + classes (AuthenticatedURL, AuthenticationException, + KerberosAuthenticator) are used from gateway-server (see + UrlConnectionDispatch); they have no javax.servlet coupling + but must be reached at the same FQNs as the upstream jar. --> <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-auth</artifactId> + <groupId>org.apache.knox</groupId> + <artifactId>gateway-shim-hadoop-auth</artifactId> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> diff --git a/gateway-shim-hadoop-auth/pom.xml b/gateway-shim-hadoop-auth/pom.xml new file mode 100644 index 000000000..ea6d5d95f --- /dev/null +++ b/gateway-shim-hadoop-auth/pom.xml @@ -0,0 +1,149 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.knox</groupId> + <artifactId>gateway</artifactId> + <version>3.0.0-SNAPSHOT</version> + </parent> + + <artifactId>gateway-shim-hadoop-auth</artifactId> + <name>gateway-shim-hadoop-auth</name> + <description> + A drop-in replacement for org.apache.hadoop:hadoop-auth's server-side + classes, rebuilt against jakarta.servlet 6.0. hadoop-auth 3.4.1 is + compiled against javax.servlet 3.1 and cannot be used directly by + Knox after the Jetty 12 / Jakarta EE 10 migration. This shim shades + the upstream jar under its original FQNs while (a) relocating every + javax.servlet reference to jakarta.servlet, and (b) rewriting the + one HttpServletResponse.setStatus(int, String) call site removed in + Servlet 6 to setStatus(int) (the reason-phrase string is dropped; + HTTP/2 does not carry it anyway). + + Modules that need AuthenticationFilter on the classpath should + depend on this instead of the upstream artifact. + </description> + + <dependencies> + <!-- Shaded into the output jar. --> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-auth</artifactId> + </dependency> + + <!-- Needed at compile time by the shaded classes after relocation. + Consumers already bring jakarta.servlet-api themselves. --> + <dependency> + <groupId>jakarta.servlet</groupId> + <artifactId>jakarta.servlet-api</artifactId> + <scope>provided</scope> + </dependency> + + <!-- ASM: used by the small post-shade patcher (Servlet6Patcher), + which drops the reason-phrase arg from setStatus(int, String) + call sites and rewrites the invoke descriptor to setStatus(I)V. --> + <dependency> + <groupId>org.ow2.asm</groupId> + <artifactId>asm</artifactId> + </dependency> + </dependencies> + + <build> + <plugins> + <!-- 1. Shade hadoop-auth into this module's jar, relocating + javax.servlet -> jakarta.servlet across every included class. + The ASF-parent-inherited version (3.1.1) ships an ASM that + cannot parse Java 17 classfiles, so we pin a newer version + locally. Once the parent pom moves past 3.5.0 this override + can be dropped. --> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + <version>3.6.0</version> + <executions> + <execution> + <id>shade-hadoop-auth</id> + <phase>package</phase> + <goals> + <goal>shade</goal> + </goals> + <configuration> + <createDependencyReducedPom>true</createDependencyReducedPom> + <shadedArtifactAttached>false</shadedArtifactAttached> + <promoteTransitiveDependencies>false</promoteTransitiveDependencies> + <artifactSet> + <includes> + <include>org.apache.hadoop:hadoop-auth</include> + </includes> + </artifactSet> + <relocations> + <relocation> + <pattern>javax.servlet</pattern> + <shadedPattern>jakarta.servlet</shadedPattern> + </relocation> + </relocations> + <!-- Drop the build-time patcher classes from the + published jar; they're only used to rewrite + the shaded hadoop-auth bytecode and would + otherwise sit in the release with dangling + org.objectweb.asm references. --> + <filters> + <filter> + <artifact>${project.groupId}:${project.artifactId}</artifact> + <excludes> + <exclude>org/apache/knox/gateway/shim/hadoopauth/Servlet6Patcher*.class</exclude> + </excludes> + </filter> + </filters> + </configuration> + </execution> + </executions> + </plugin> + + <!-- 2. Post-shade: rewrite setStatus(int, String) -> setStatus(int) + inside the shaded classes. Runs against target/classes for + the patcher, and against the primary artifact for the patch + target. --> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <version>${exec-maven-plugin.version}</version> + <executions> + <execution> + <id>patch-servlet6-removed-methods</id> + <phase>package</phase> + <goals> + <goal>java</goal> + </goals> + <configuration> + <mainClass>org.apache.knox.gateway.shim.hadoopauth.Servlet6Patcher</mainClass> + <arguments> + <argument>${project.build.directory}/${project.build.finalName}.jar</argument> + </arguments> + <includePluginDependencies>false</includePluginDependencies> + <includeProjectDependencies>true</includeProjectDependencies> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> +</project> diff --git a/gateway-shim-hadoop-auth/src/main/java/org/apache/knox/gateway/shim/hadoopauth/Servlet6Patcher.java b/gateway-shim-hadoop-auth/src/main/java/org/apache/knox/gateway/shim/hadoopauth/Servlet6Patcher.java new file mode 100644 index 000000000..0b6552ad0 --- /dev/null +++ b/gateway-shim-hadoop-auth/src/main/java/org/apache/knox/gateway/shim/hadoopauth/Servlet6Patcher.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.shim.hadoopauth; + +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.ClassWriter; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Opcodes; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.util.Enumeration; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.jar.JarOutputStream; + +/** + * Rewrites invocations of {@code jakarta.servlet.http.HttpServletResponse.setStatus(int, String)} + * inside a shaded jar into {@code setStatus(int)}. The 2-arg form was removed in Servlet 6; + * hadoop-auth 3.4.1 still calls it in {@code AuthenticationFilter} on the auth-failure path, + * so the shaded classes would throw {@code NoSuchMethodError} at runtime otherwise. + * + * The rewrite drops the reason-phrase argument (the top-of-stack {@code String}) with a {@code POP} + * and rewrites the {@code INVOKEINTERFACE} descriptor from {@code (ILjava/lang/String;)V} to + * {@code (I)V}. The reason phrase is not carried over HTTP/2 anyway. + * + * Usage: {@code Servlet6Patcher path/to/shaded.jar}. The jar is rewritten in place. + */ +public final class Servlet6Patcher { + + private static final String HTTP_SERVLET_RESPONSE = "jakarta/servlet/http/HttpServletResponse"; + private static final String SET_STATUS = "setStatus"; + private static final String OLD_DESCRIPTOR = "(ILjava/lang/String;)V"; + private static final String NEW_DESCRIPTOR = "(I)V"; + + public static void main(String[] args) throws IOException { + if (args.length != 1) { + System.err.println("Usage: Servlet6Patcher <jar>"); + System.exit(2); + } + Path jar = Paths.get(args[0]); + Path tmp = Files.createTempFile("shim-", ".jar"); + int patchedInvocations = 0; + int patchedClasses = 0; + try (JarFile src = new JarFile(jar.toFile()); + OutputStream out = Files.newOutputStream(tmp); + JarOutputStream dst = new JarOutputStream(out)) { + Enumeration<JarEntry> entries = src.entries(); + while (entries.hasMoreElements()) { + JarEntry entry = entries.nextElement(); + JarEntry copy = new JarEntry(entry.getName()); + copy.setTime(entry.getTime()); + dst.putNextEntry(copy); + if (entry.isDirectory() || !entry.getName().endsWith(".class")) { + try (InputStream in = src.getInputStream(entry)) { + in.transferTo(dst); + } + dst.closeEntry(); + continue; + } + byte[] original; + try (InputStream in = src.getInputStream(entry)) { + original = in.readAllBytes(); + } + PatchingVisitor visitor = new PatchingVisitor(); + byte[] rewritten = visitor.rewrite(original); + if (visitor.hits > 0) { + patchedInvocations += visitor.hits; + patchedClasses += 1; + dst.write(rewritten); + } else { + dst.write(original); + } + dst.closeEntry(); + } + } + Files.move(tmp, jar, StandardCopyOption.REPLACE_EXISTING); + System.out.println("Servlet6Patcher: rewrote " + patchedInvocations + + " setStatus(int, String) call(s) across " + patchedClasses + " class(es)"); + if (patchedInvocations == 0) { + // Defensive: if a future hadoop-auth upgrade already removed this call, the patch is a + // no-op and safe to keep. If it's ever found to be missing when we expected it, the shim + // was still built correctly; log-only, don't fail the build. + System.out.println("Servlet6Patcher: nothing to patch (upstream may have already dropped " + + "setStatus(int, String))."); + } + } + + /** + * Visits every method in a class and rewrites {@code setStatus(int, String)} into + * {@code setStatus(int)} preceded by a {@code POP} of the reason-phrase argument. + */ + static final class PatchingVisitor extends ClassVisitor { + int hits; + + PatchingVisitor() { + super(Opcodes.ASM9); + } + + byte[] rewrite(byte[] classBytes) { + ClassReader reader = new ClassReader(classBytes); + ClassWriter writer = new ClassWriter(reader, 0); + this.cv = writer; + reader.accept(this, 0); + return writer.toByteArray(); + } + + @Override + public MethodVisitor visitMethod(int access, String name, String descriptor, + String signature, String[] exceptions) { + MethodVisitor next = super.visitMethod(access, name, descriptor, signature, exceptions); + return new PatchingMethodVisitor(next); + } + + private final class PatchingMethodVisitor extends MethodVisitor { + PatchingMethodVisitor(MethodVisitor delegate) { + super(Opcodes.ASM9, delegate); + } + + @Override + public void visitMethodInsn(int opcode, String owner, String mName, String mDescriptor, + boolean isInterface) { + if ((opcode == Opcodes.INVOKEINTERFACE || opcode == Opcodes.INVOKEVIRTUAL) + && HTTP_SERVLET_RESPONSE.equals(owner) + && SET_STATUS.equals(mName) + && OLD_DESCRIPTOR.equals(mDescriptor)) { + // Stack before: ..., response, statusCode, reasonPhrase + // We want: ..., response, statusCode then INVOKE... setStatus(I)V + super.visitInsn(Opcodes.POP); + super.visitMethodInsn(opcode, owner, mName, NEW_DESCRIPTOR, isInterface); + hits++; + return; + } + super.visitMethodInsn(opcode, owner, mName, mDescriptor, isInterface); + } + } + } + + private Servlet6Patcher() { + // utility + } +} diff --git a/gateway-topology-hadoop-xml/pom.xml b/gateway-topology-hadoop-xml/pom.xml index 9930ab09a..ae3e91e1b 100644 --- a/gateway-topology-hadoop-xml/pom.xml +++ b/gateway-topology-hadoop-xml/pom.xml @@ -62,6 +62,7 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-auth</artifactId> + <scope>test</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> diff --git a/pom.xml b/pom.xml index 888f759f8..b0a725347 100644 --- a/pom.xml +++ b/pom.xml @@ -72,6 +72,7 @@ <module>gateway-spi</module> <module>gateway-spi-common</module> <module>gateway-shim-opensaml-security-api</module> + <module>gateway-shim-hadoop-auth</module> <module>gateway-discovery-ambari</module> <module>gateway-discovery-cm</module> <module>gateway-performance-test</module> @@ -1832,6 +1833,14 @@ <groupId>org.slf4j</groupId> <artifactId>slf4j-reload4j</artifactId> </exclusion> + <!-- hadoop-auth 3.4.1 is compiled against javax.servlet 3.1. + Knox depends on gateway-shim-hadoop-auth instead, which + republishes the classes rebuilt against jakarta.servlet 6. + Keep the javax.* API off the compile classpath. --> + <exclusion> + <groupId>javax.servlet</groupId> + <artifactId>javax.servlet-api</artifactId> + </exclusion> </exclusions> </dependency> @@ -2760,6 +2769,23 @@ </exclusion> </exclusions> </dependency> + <!-- Jakarta shim: shaded hadoop-auth, with every javax.servlet + reference relocated to jakarta.servlet and the Servlet 6- + removed HttpServletResponse.setStatus(int, String) call + patched down to setStatus(int). Modules that need + AuthenticationFilter should depend on this instead of + org.apache.hadoop:hadoop-auth. --> + <dependency> + <groupId>org.apache.knox</groupId> + <artifactId>gateway-shim-hadoop-auth</artifactId> + <version>${project.version}</version> + <exclusions> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-auth</artifactId> + </exclusion> + </exclusions> + </dependency> <dependency> <groupId>org.opensaml</groupId> <artifactId>opensaml-security-impl</artifactId>
