This is an automated email from the ASF dual-hosted git repository.
abhishek pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 6aad9b08ddc Fix low sonatype findings (#17017)
6aad9b08ddc is described below
commit 6aad9b08ddcf56c29078a730eadd59c55f0f9c6b
Author: Misha <[email protected]>
AuthorDate: Mon Sep 16 12:40:25 2024 +0200
Fix low sonatype findings (#17017)
Fixed vulnerabilities
CVE-2021-26291 : Apache Maven is vulnerable to Man-in-the-Middle (MitM)
attacks. Various
functions across several files, mentioned below, allow for custom
repositories to use the
insecure HTTP protocol. An attacker can exploit this as part of a
Man-in-the-Middle (MitM)
attack, taking over or impersonating a repository using the insecure HTTP
protocol.
Unsuspecting users may then have the compromised repository defined as a
dependency in
their Project Object Model (pom) file and download potentially malicious
files from it.
Was fixed by removing outdated tesla-aether library containing vulnerable
maven-settings (v3.1.1) package, pull-deps utility updated to use maven
resolver instead.
sonatype-2020-0244 : The joni package is vulnerable to Man-in-the-Middle
(MitM) attacks.
This project downloads dependencies over HTTP due to an insecure repository
configuration
within the .pom file. Consequently, a MitM could intercept requests to the
specified
repository and replace the requested dependencies with malicious versions,
which can execute
arbitrary code from the application that was built with them.
Was fixed by upgrading joni package to recommended 2.1.34 version
---
indexing-service/pom.xml | 13 +-
.../overlord/sampler/InputSourceSamplerTest.java | 2 +-
licenses.yaml | 110 +++++--
pom.xml | 5 +
server/pom.xml | 19 +-
services/pom.xml | 54 +++-
.../org/apache/druid/cli/PullDependencies.java | 360 +++++++--------------
.../org/apache/druid/cli/PullDependenciesTest.java | 247 ++++++++++++--
8 files changed, 491 insertions(+), 319 deletions(-)
diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml
index c851c6ef98f..a9479d7db08 100644
--- a/indexing-service/pom.xml
+++ b/indexing-service/pom.xml
@@ -189,10 +189,6 @@
<artifactId>datasketches-memory</artifactId>
<scope>provided</scope>
</dependency>
- <dependency>
- <groupId>net.thisptr</groupId>
- <artifactId>jackson-jq</artifactId>
- </dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-core-asl</artifactId>
@@ -203,10 +199,6 @@
<artifactId>commons-collections4</artifactId>
<scope>provided</scope>
</dependency>
- <dependency>
- <groupId>org.eclipse.aether</groupId>
- <artifactId>aether-api</artifactId>
- </dependency>
<!-- Tests -->
<dependency>
<groupId>junit</groupId>
@@ -267,6 +259,11 @@
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.maven.resolver</groupId>
+ <artifactId>maven-resolver-api</artifactId>
+ <version>1.3.1</version>
+ </dependency>
</dependencies>
<profiles>
diff --git
a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/sampler/InputSourceSamplerTest.java
b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/sampler/InputSourceSamplerTest.java
index 80d88e0be17..690b74e347e 100644
---
a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/sampler/InputSourceSamplerTest.java
+++
b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/sampler/InputSourceSamplerTest.java
@@ -23,7 +23,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
-import net.thisptr.jackson.jq.internal.misc.Lists;
+import com.google.common.collect.Lists;
import org.apache.druid.client.indexing.SamplerResponse;
import org.apache.druid.client.indexing.SamplerResponse.SamplerResponseRow;
import org.apache.druid.data.input.InputFormat;
diff --git a/licenses.yaml b/licenses.yaml
index c4e2fa52300..0ecffdb0a0a 100644
--- a/licenses.yaml
+++ b/licenses.yaml
@@ -1837,33 +1837,38 @@ name: Apache Maven
license_category: binary
module: java-core
license_name: Apache License version 2.0
-version: 3.1.1
+version: 3.6.0
+libraries:
+ - org.apache.maven: maven-repository-metadata
+ - org.apache.maven: maven-builder-support
+notices:
+ - maven-repository-metadata: |
+ Maven Repository Metadata Model
+ Copyright 2001-2018 The Apache Software Foundation
+ - maven-builder-support: |
+ Maven Builder Support
+ Copyright 2001-2018 The Apache Software Foundation
+---
+
+name: Maven Artifact Resolver Provider
+license_category: binary
+module: java-core
+license_name: Apache License version 2.0
+version: 3.6.0
libraries:
- - org.apache.maven: maven-aether-provider
+ - org.apache.maven: maven-resolver-provider
- org.apache.maven: maven-model
- org.apache.maven: maven-model-builder
- - org.apache.maven: maven-repository-metadata
- - org.apache.maven: maven-settings
- - org.apache.maven: maven-settings-builder
notices:
- - maven-aether-provider: |
- Maven Aether Provider
- Copyright 2001-2013 The Apache Software Foundation
+ - maven-resolver-provider: |
+ Maven Artifact Resolver Provider
+ Copyright 2001-2018 The Apache Software Foundation
- maven-model: |
Maven Model
- Copyright 2001-2013 The Apache Software Foundation
+ Copyright 2001-2018 The Apache Software Foundation
- maven-model-builder: |
Maven Model Builder
- Copyright 2001-2013 The Apache Software Foundation
- - maven-repository-metadata: |
- Maven Repository Metadata Model
- Copyright 2001-2013 The Apache Software Foundation
- - maven-settings: |
- Maven Settings
- Copyright 2001-2013 The Apache Software Foundation
- - maven-settings-builder: |
- Maven Settings Builder
- Copyright 2001-2013 The Apache Software Foundation
+ Copyright 2001-2018 The Apache Software Foundation
---
name: Apache Maven Artifact
@@ -1879,6 +1884,67 @@ notices:
Copyright 2001-2018 The Apache Software Foundation
---
+name: Maven Artifact Resolver Connector Basic
+license_category: binary
+module: java-core
+license_name: Apache License version 2.0
+version: 1.3.1
+libraries:
+ - org.apache.maven.resolver: maven-resolver-connector-basic
+ - org.apache.maven.resolver: maven-resolver-spi
+ - org.apache.maven.resolver: maven-resolver-api
+ - org.apache.maven.resolver: maven-resolver-util
+notices:
+ - maven-resolver-connector-basic: |
+ Maven Artifact Resolver Connector Basic
+ Copyright 2001-2018 The Apache Software Foundation
+ - maven-resolver-spi: |
+ Maven Artifact Resolver SPI
+ Copyright 2001-2018 The Apache Software Foundation
+ - maven-resolver-api: |
+ Maven Artifact Resolver API
+ Copyright 2001-2018 The Apache Software Foundation
+ - maven-resolver-util: |
+ Maven Artifact Resolver Utilities
+ Copyright 2001-2018 The Apache Software Foundation
+---
+
+name: Maven Artifact Resolver Transport HTTP
+license_category: binary
+module: java-core
+license_name: Apache License version 2.0
+version: 1.3.1
+libraries:
+ - org.apache.maven.resolver: maven-resolver-transport-http
+notices:
+ - maven-resolver-transport-http: |
+ Maven Artifact Resolver Transport HTTP
+ Copyright 2001-2018 The Apache Software Foundation
+
+---
+
+name: Maven Artifact Resolver Implementation
+license_category: binary
+module: java-core
+license_name: Apache License version 2.0
+version: 1.3.1
+libraries:
+ - org.apache.maven.resolver: maven-resolver-impl
+notices:
+ - maven-resolver-impl: |
+ Maven Artifact Resolver Implementation
+ Copyright 2001-2018 The Apache Software Foundation
+---
+
+name: Plexus Component Annotations
+license_category: binary
+module: java-core
+license_name: Apache License version 2.0
+version: 1.7.1
+libraries:
+ - org.codehaus.plexus: plexus-component-annotations
+---
+
name: Apache Maven Wagon API
license_category: binary
module: java-core
@@ -1967,7 +2033,7 @@ name: Plexus Interpolation API
license_category: binary
module: java-core
license_name: Apache License version 2.0
-version: 1.19
+version: 1.25
libraries:
- org.codehaus.plexus: plexus-interpolation
@@ -3245,7 +3311,7 @@ name: JCodings
license_category: binary
module: java-core
license_name: MIT License
-version: 1.0.43
+version: 1.0.50
copyright: JRuby Team
license_file_path: licenses/bin/jcodings.MIT
libraries:
@@ -3257,7 +3323,7 @@ name: Joni
license_category: binary
module: java-core
license_name: MIT License
-version: 2.1.27
+version: 2.1.34
copyright: JRuby Team
license_file_path: licenses/bin/joni.MIT
libraries:
diff --git a/pom.xml b/pom.xml
index d8e96ed859d..42f7333ea2c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1363,6 +1363,11 @@
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>org.jruby.joni</groupId>
+ <artifactId>joni</artifactId>
+ <version>2.1.34</version>
+ </dependency>
</dependencies>
</dependencyManagement>
diff --git a/server/pom.xml b/server/pom.xml
index d1954adc319..6cfeee55f57 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -140,8 +140,19 @@
<artifactId>jsr305</artifactId>
</dependency>
<dependency>
- <groupId>io.tesla.aether</groupId>
- <artifactId>tesla-aether</artifactId>
+ <groupId>org.apache.maven.resolver</groupId>
+ <artifactId>maven-resolver-connector-basic</artifactId>
+ <version>1.3.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven.resolver</groupId>
+ <artifactId>maven-resolver-transport-http</artifactId>
+ <version>1.3.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven</groupId>
+ <artifactId>maven-resolver-provider</artifactId>
+ <version>3.6.0</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
@@ -500,7 +511,9 @@
<configuration>
<usedDependencies>
<!-- These are needed for scope: compile -->
- <dependency>io.tesla.aether:tesla-aether</dependency>
+
<dependency>org.apache.maven:maven-resolver-provider</dependency>
+
<dependency>org.apache.maven.resolver:maven-resolver-transport-http</dependency>
+
<dependency>org.apache.maven.resolver:maven-resolver-connector-basic</dependency>
<!-- These are needed for scope: runtime -->
<dependency>org.xerial.snappy:snappy-java</dependency>
</usedDependencies>
diff --git a/services/pom.xml b/services/pom.xml
index 2f8b70bd88e..83c109b5aba 100644
--- a/services/pom.xml
+++ b/services/pom.xml
@@ -56,6 +56,12 @@
<groupId>org.apache.druid</groupId>
<artifactId>druid-indexing-service</artifactId>
<version>${project.parent.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.eclipse.aether</groupId>
+ <artifactId>aether-api</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.druid</groupId>
@@ -171,10 +177,6 @@
<groupId>io.netty</groupId>
<artifactId>netty-common</artifactId>
</dependency>
- <dependency>
- <groupId>org.eclipse.aether</groupId>
- <artifactId>aether-api</artifactId>
- </dependency>
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
@@ -187,17 +189,49 @@
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
- <dependency>
- <groupId>org.eclipse.aether</groupId>
- <artifactId>aether-util</artifactId>
- </dependency>
<dependency>
<groupId>com.google.inject.extensions</groupId>
<artifactId>guice-servlet</artifactId>
</dependency>
<dependency>
- <groupId>io.tesla.aether</groupId>
- <artifactId>tesla-aether</artifactId>
+ <groupId>org.apache.maven.resolver</groupId>
+ <artifactId>maven-resolver-connector-basic</artifactId>
+ <version>1.3.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven.resolver</groupId>
+ <artifactId>maven-resolver-transport-http</artifactId>
+ <version>1.3.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven.resolver</groupId>
+ <artifactId>maven-resolver-util</artifactId>
+ <version>1.3.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven.resolver</groupId>
+ <artifactId>maven-resolver-impl</artifactId>
+ <version>1.3.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven.resolver</groupId>
+ <artifactId>maven-resolver-spi</artifactId>
+ <version>1.3.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven.resolver</groupId>
+ <artifactId>maven-resolver-api</artifactId>
+ <version>1.3.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven</groupId>
+ <artifactId>maven-artifact</artifactId>
+ <version>3.6.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven</groupId>
+ <artifactId>maven-resolver-provider</artifactId>
+ <version>3.6.0</version>
</dependency>
<dependency>
<groupId>javax.xml.bind</groupId>
diff --git a/services/src/main/java/org/apache/druid/cli/PullDependencies.java
b/services/src/main/java/org/apache/druid/cli/PullDependencies.java
index 6ea8626ba7e..f0588d2b372 100644
--- a/services/src/main/java/org/apache/druid/cli/PullDependencies.java
+++ b/services/src/main/java/org/apache/druid/cli/PullDependencies.java
@@ -22,43 +22,45 @@ package org.apache.druid.cli;
import com.github.rvesse.airline.annotations.Command;
import com.github.rvesse.airline.annotations.Option;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSetMultimap;
import com.google.common.collect.SetMultimap;
import com.google.inject.Inject;
-import io.netty.util.SuppressForbidden;
-import io.tesla.aether.Repository;
-import io.tesla.aether.TeslaAether;
-import io.tesla.aether.guice.RepositorySystemSessionProvider;
-import io.tesla.aether.internal.DefaultTeslaAether;
import org.apache.druid.guice.ExtensionsConfig;
import org.apache.druid.indexing.common.config.TaskConfig;
import org.apache.druid.java.util.common.FileUtils;
import org.apache.druid.java.util.common.ISE;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.java.util.common.logger.Logger;
+import org.apache.maven.artifact.resolver.ArtifactNotFoundException;
+import org.apache.maven.repository.internal.MavenRepositorySystemUtils;
+import org.eclipse.aether.DefaultRepositorySystemSession;
+import org.eclipse.aether.RepositorySystem;
import org.eclipse.aether.RepositorySystemSession;
import org.eclipse.aether.artifact.Artifact;
import org.eclipse.aether.artifact.DefaultArtifact;
import org.eclipse.aether.collection.CollectRequest;
+import org.eclipse.aether.connector.basic.BasicRepositoryConnectorFactory;
import org.eclipse.aether.graph.Dependency;
import org.eclipse.aether.graph.DependencyNode;
-import org.eclipse.aether.repository.Authentication;
+import org.eclipse.aether.impl.DefaultServiceLocator;
+import org.eclipse.aether.repository.LocalRepository;
import org.eclipse.aether.repository.Proxy;
import org.eclipse.aether.repository.RemoteRepository;
+import org.eclipse.aether.resolution.ArtifactResult;
import org.eclipse.aether.resolution.DependencyRequest;
+import org.eclipse.aether.resolution.DependencyResolutionException;
+import org.eclipse.aether.resolution.DependencyResult;
+import org.eclipse.aether.spi.connector.RepositoryConnectorFactory;
+import org.eclipse.aether.spi.connector.transport.TransporterFactory;
+import org.eclipse.aether.transport.http.HttpTransporterFactory;
import org.eclipse.aether.util.artifact.JavaScopes;
import org.eclipse.aether.util.filter.DependencyFilterUtils;
import org.eclipse.aether.util.repository.AuthenticationBuilder;
+import org.eclipse.aether.util.repository.DefaultProxySelector;
import java.io.File;
import java.io.IOException;
-import java.io.OutputStream;
-import java.io.PrintStream;
-import java.io.UnsupportedEncodingException;
-import java.net.URI;
-import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
@@ -86,72 +88,6 @@ public class PullDependencies implements Runnable
.put("com.fasterxml.jackson.core", "jackson-core")
.put("com.fasterxml.jackson.core", "jackson-annotations")
.build();
- /*
- It is possible that extensions will pull down a lot of jars that are
either
- duplicates OR conflict with druid jars. In that case, there are two
problems that arise
-
- 1. Large quantity of jars are passed around to things like hadoop when
they are not needed (and should not be included)
- 2. Classpath priority becomes "mostly correct" and attempted to enforced
correctly, but not fully tested
-
- These jar groups should be included by druid and *not* pulled down in
extensions
- Note to future developers: This list is hand-crafted and will probably
be out of date in the future
- A good way to know where to look for errant dependencies is to compare
the lib/ directory in the distribution
- tarball with the jars included in the extension directories.
-
- This list is best-effort, and might still pull down more than desired.
-
- A simple example is that if an extension's dependency uses
some-library-123.jar,
- druid uses some-library-456.jar, and hadoop uses some-library-666.jar,
then we probably want to use some-library-456.jar,
- so don't pull down some-library-123.jar, and ask hadoop to load
some-library-456.jar.
-
- In the case where some-library is NOT on this list, both
some-library-456.jar and some-library-123.jar will be
- on the class path and propagated around the system. Most places TRY to
make sure some-library-456.jar has
- precedence, but it is easy for this assumption to be violated and for
the precedence of some-library-456.jar,
- some-library-123.jar and some-library-456.jar to not be properly defined.
-
- As of this writing there are no special unit tests for classloader
issues and library version conflicts.
-
- Different tasks which are classloader sensitive attempt to maintain a
sane order for loading libraries in the
- classloader, but it is always possible that something didn't load in the
right order. Also we don't want to be
- throwing around a ton of jars we don't need to.
-
- Here is a list of dependencies extensions should probably exclude.
-
- Conflicts can be discovered using the following command on the
distribution tarball:
- `find lib -iname "*.jar" | cut -d / -f 2 | sed -e
's/-[0-9]\.[0-9]/@/' | cut -f 1 -d @ | sort | uniq | xargs -I {} find
extensions -name "*{}*.jar" | sort`
-
- "org.apache.druid",
- "com.metamx.druid",
- "asm",
- "org.ow2.asm",
- "org.jboss.netty",
- "com.google.guava",
- "com.google.code.findbugs",
- "com.google.protobuf",
- "com.esotericsoftware.minlog",
- "log4j",
- "org.slf4j",
- "commons-logging",
- "org.eclipse.jetty",
- "org.mortbay.jetty",
- "com.sun.jersey",
- "com.sun.jersey.contribs",
- "common-beanutils",
- "commons-codec",
- "commons-lang",
- "commons-cli",
- "commons-io",
- "javax.activation",
- "org.apache.httpcomponents",
- "org.apache.zookeeper",
- "org.codehaus.jackson",
- "com.fasterxml.jackson",
- "com.fasterxml.jackson.core",
- "com.fasterxml.jackson.dataformat",
- "com.fasterxml.jackson.datatype",
- "org.roaringbitmap",
- "net.java.dev.jets3t"
- */
private static final Dependencies SECURITY_VULNERABILITY_EXCLUSIONS =
Dependencies.builder()
@@ -160,8 +96,6 @@ public class PullDependencies implements Runnable
private final Dependencies hadoopExclusions;
- private TeslaAether aether;
-
@Inject
public ExtensionsConfig extensionsConfig;
@@ -196,60 +130,53 @@ public class PullDependencies implements Runnable
title = "A local repository that Maven will use to put downloaded files.
Then pull-deps will lay these files out into the extensions directory as
needed."
)
public String localRepository = StringUtils.format("%s/%s",
System.getProperty("user.home"), ".m2/repository");
-
- @Option(
- name = {"-r", "--remoteRepository"},
- title = "Add a remote repository. Unless
--no-default-remote-repositories is provided, these will be used after
https://repo1.maven.org/maven2/";
- )
- List<String> remoteRepositories = new ArrayList<>();
-
@Option(
name = "--no-default-remote-repositories",
description = "Don't use the default remote repositories, only use the
repositories provided directly via --remoteRepository"
)
public boolean noDefaultRemoteRepositories = false;
-
@Option(
name = {"-d", "--defaultVersion"},
title = "Version to use for extension artifacts without version
information."
)
public String defaultVersion =
PullDependencies.class.getPackage().getImplementationVersion();
-
@Option(
name = {"--use-proxy"},
title = "Use http/https proxy to pull dependencies."
)
public boolean useProxy = false;
-
@Option(
name = {"--proxy-type"},
title = "The proxy type, should be either http or https"
)
public String proxyType = "https";
-
@Option(
name = {"--proxy-host"},
title = "The proxy host"
)
public String proxyHost = "";
-
@Option(
name = {"--proxy-port"},
title = "The proxy port"
)
public int proxyPort = -1;
-
@Option(
name = {"--proxy-username"},
title = "The proxy username"
)
public String proxyUsername = "";
-
@Option(
name = {"--proxy-password"},
title = "The proxy password"
)
public String proxyPassword = "";
+ @Option(
+ name = {"-r", "--remoteRepository"},
+ title = "Add a remote repository. Unless
--no-default-remote-repositories is provided, these will be used after
https://repo1.maven.org/maven2/";
+ )
+ List<String> remoteRepositories = new ArrayList<>();
+ private RepositorySystem repositorySystem;
+ private RepositorySystemSession repositorySystemSession;
@SuppressWarnings("unused") // used by com.github.rvesse.airline
public PullDependencies()
@@ -261,18 +188,74 @@ public class PullDependencies implements Runnable
}
// Used for testing only
- PullDependencies(TeslaAether aether, ExtensionsConfig extensionsConfig,
Dependencies hadoopExclusions)
+ PullDependencies(
+ RepositorySystem repositorySystem,
+ RepositorySystemSession repositorySystemSession,
+ ExtensionsConfig extensionsConfig,
+ Dependencies hadoopExclusions
+ )
{
- this.aether = aether;
+ this.repositorySystem = repositorySystem;
+ this.repositorySystemSession = repositorySystemSession;
this.extensionsConfig = extensionsConfig;
this.hadoopExclusions = hadoopExclusions;
}
+ private RepositorySystem getRepositorySystem()
+ {
+ DefaultServiceLocator locator =
MavenRepositorySystemUtils.newServiceLocator();
+ locator.addService(RepositoryConnectorFactory.class,
BasicRepositoryConnectorFactory.class);
+ locator.addService(TransporterFactory.class, HttpTransporterFactory.class);
+ return locator.getService(RepositorySystem.class);
+ }
+
+ protected RepositorySystemSession getRepositorySystemSession()
+ {
+ DefaultRepositorySystemSession session =
MavenRepositorySystemUtils.newSession();
+ LocalRepository localRepo = new LocalRepository(localRepository);
+
session.setLocalRepositoryManager(repositorySystem.newLocalRepositoryManager(session,
localRepo));
+
+ // Set up the proxy configuration if required
+ if (useProxy) {
+ Proxy proxy = new Proxy(
+ proxyType,
+ proxyHost,
+ proxyPort,
+ isBlank(proxyUsername) ? null : new AuthenticationBuilder()
+ .addUsername(proxyUsername)
+ .addPassword(proxyPassword)
+ .build()
+ );
+
+ final DefaultProxySelector proxySelector = new DefaultProxySelector();
+ proxySelector.add(proxy, null);
+
+ session.setProxySelector(proxySelector);
+ }
+
+ return session;
+ }
+
+ protected List<RemoteRepository> getRemoteRepositories()
+ {
+ List<RemoteRepository> repositories = new ArrayList<>();
+
+ if (!noDefaultRemoteRepositories) {
+ repositories.add(new RemoteRepository.Builder("central", "default",
DEFAULT_REMOTE_REPOSITORIES.get(0)).build());
+ }
+
+ for (String repoUrl : remoteRepositories) {
+ repositories.add(new RemoteRepository.Builder(null, "default",
repoUrl).build());
+ }
+
+ return repositories;
+ }
+
@Override
public void run()
{
- if (aether == null) {
- aether = getAetherClient();
+ if (repositorySystem == null) {
+ repositorySystem = getRepositorySystem();
}
final File extensionsDir = new File(extensionsConfig.getDirectory());
@@ -334,7 +317,7 @@ public class PullDependencies implements Runnable
}
}
- private Artifact getArtifact(String coordinate)
+ protected Artifact getArtifact(String coordinate)
{
DefaultArtifact versionedArtifact;
try {
@@ -367,6 +350,12 @@ public class PullDependencies implements Runnable
{
final CollectRequest collectRequest = new CollectRequest();
collectRequest.setRoot(new Dependency(versionedArtifact,
JavaScopes.RUNTIME));
+
+ List<RemoteRepository> repositories = getRemoteRepositories();
+ for (RemoteRepository repo : repositories) {
+ collectRequest.addRepository(repo);
+ }
+
final DependencyRequest dependencyRequest = new DependencyRequest(
collectRequest,
DependencyFilterUtils.andFilter(
@@ -375,13 +364,7 @@ public class PullDependencies implements Runnable
String scope = node.getDependency().getScope();
if (scope != null) {
scope = StringUtils.toLowerCase(scope);
- if ("provided".equals(scope)) {
- return false;
- }
- if ("test".equals(scope)) {
- return false;
- }
- if ("system".equals(scope)) {
+ if ("provided".equals(scope) || "test".equals(scope) ||
"system".equals(scope)) {
return false;
}
}
@@ -402,7 +385,17 @@ public class PullDependencies implements Runnable
try {
log.info("Start downloading extension [%s]", versionedArtifact);
- final List<Artifact> artifacts =
aether.resolveArtifacts(dependencyRequest);
+ if (repositorySystemSession == null) {
+ repositorySystemSession = getRepositorySystemSession();
+ }
+
+ final DependencyResult result = repositorySystem.resolveDependencies(
+ repositorySystemSession,
+ dependencyRequest
+ );
+ final List<Artifact> artifacts = result.getArtifactResults().stream()
+ .map(ArtifactResult::getArtifact)
+ .collect(Collectors.toList());
for (Artifact artifact : artifacts) {
if (exclusions.contain(artifact)) {
@@ -413,138 +406,18 @@ public class PullDependencies implements Runnable
}
}
}
- catch (Exception e) {
- log.error(e, "Unable to resolve artifacts for [%s].", dependencyRequest);
- throw new RuntimeException(e);
- }
- log.info("Finish downloading extension [%s]", versionedArtifact);
- }
-
- @SuppressForbidden(reason = "System#out")
- private DefaultTeslaAether getAetherClient()
- {
- /*
- DefaultTeslaAether logs a bunch of stuff to System.out, which is annoying.
We choose to disable that
- unless debug logging is turned on. "Disabling" it, however, is kinda
bass-ackwards. We copy out a reference
- to the current System.out, and set System.out to a noop output stream.
Then after DefaultTeslaAether has pulled
- The reference we swap things back.
-
- This has implications for other things that are running in parallel to
this. Namely, if anything else also grabs
- a reference to System.out or tries to log to it while we have things
adjusted like this, then they will also log
- to nothingness. Fortunately, the code that calls this is single-threaded
and shouldn't hopefully be running
- alongside anything else that's grabbing System.out. But who knows.
- */
-
- final List<String> remoteUriList = new ArrayList<>();
- if (!noDefaultRemoteRepositories) {
- remoteUriList.addAll(DEFAULT_REMOTE_REPOSITORIES);
- }
- remoteUriList.addAll(remoteRepositories);
-
- List<Repository> remoteRepositories = new ArrayList<>();
- for (String uri : remoteUriList) {
- try {
- URI u = new URI(uri);
- Repository r = new Repository(uri);
-
- if (u.getUserInfo() != null) {
- String[] auth = u.getUserInfo().split(":", 2);
- if (auth.length == 2) {
- r.setUsername(auth[0]);
- r.setPassword(auth[1]);
- } else {
- log.warn(
- "Invalid credentials in repository URI, expecting
[<user>:<password>], got [%s] for [%s]",
- u.getUserInfo(),
- uri
- );
- }
- }
- remoteRepositories.add(r);
- }
- catch (URISyntaxException e) {
- throw new RuntimeException(e);
+ catch (DependencyResolutionException e) {
+ if (e.getCause() instanceof ArtifactNotFoundException) {
+ log.error("Artifact not found in any configured repositories: [%s]",
versionedArtifact);
+ } else {
+ log.error(e, "Unable to resolve artifacts for [%s].",
dependencyRequest);
}
}
-
- if (log.isTraceEnabled() || log.isDebugEnabled()) {
- return createTeslaAether(remoteRepositories);
- }
-
- PrintStream oldOut = System.out;
- try {
- System.setOut(
- new PrintStream(
- new OutputStream()
- {
- @Override
- public void write(int b)
- {
-
- }
-
- @Override
- public void write(byte[] b)
- {
-
- }
-
- @Override
- public void write(byte[] b, int off, int len)
- {
-
- }
- },
- false,
- StringUtils.UTF8_STRING
- )
- );
- return createTeslaAether(remoteRepositories);
- }
- catch (UnsupportedEncodingException e) {
- // should never happen
- throw new IllegalStateException(e);
- }
- finally {
- System.setOut(oldOut);
- }
- }
-
- private DefaultTeslaAether createTeslaAether(List<Repository>
remoteRepositories)
- {
- if (!useProxy) {
- return new DefaultTeslaAether(
- localRepository,
- remoteRepositories.toArray(new Repository[0])
- );
- }
-
- if (!StringUtils.toLowerCase(proxyType).equals(Proxy.TYPE_HTTP) &&
- !StringUtils.toLowerCase(proxyType).equals(Proxy.TYPE_HTTPS)) {
- throw new IllegalArgumentException("invalid proxy type: " + proxyType);
+ catch (IOException e) {
+ log.error(e, "I/O error while processing artifact [%s].",
versionedArtifact);
+ throw new RuntimeException(e);
}
-
- RepositorySystemSession repositorySystemSession =
- new RepositorySystemSessionProvider(new File(localRepository)).get();
- List<RemoteRepository> rl = remoteRepositories.stream().map(r -> {
- RemoteRepository.Builder builder = new
RemoteRepository.Builder(r.getId(), "default", r.getUrl());
- if (r.getUsername() != null && r.getPassword() != null) {
- Authentication auth = new
AuthenticationBuilder().addUsername(r.getUsername())
-
.addPassword(r.getPassword())
- .build();
- builder.setAuthentication(auth);
- }
-
- final Authentication proxyAuth;
- if (Strings.isNullOrEmpty(proxyUsername)) {
- proxyAuth = null;
- } else {
- proxyAuth = new
AuthenticationBuilder().addUsername(proxyUsername).addPassword(proxyPassword).build();
- }
- builder.setProxy(new Proxy(proxyType, proxyHost, proxyPort, proxyAuth));
- return builder.build();
- }).collect(Collectors.toList());
- return new DefaultTeslaAether(rl, repositorySystemSession);
+ log.info("Finish downloading extension [%s]", versionedArtifact);
}
/**
@@ -567,6 +440,11 @@ public class PullDependencies implements Runnable
}
}
+ private boolean isBlank(final String toCheck)
+ {
+ return toCheck == null || toCheck.isEmpty();
+ }
+
@VisibleForTesting
static class Dependencies
{
@@ -579,15 +457,15 @@ public class PullDependencies implements Runnable
groupIdToArtifactIds = builder.groupIdToArtifactIdsBuilder.build();
}
- boolean contain(Artifact artifact)
+ static Builder builder()
{
- Set<String> artifactIds =
groupIdToArtifactIds.get(artifact.getGroupId());
- return artifactIds.contains(ANY_ARTIFACT_ID) ||
artifactIds.contains(artifact.getArtifactId());
+ return new Builder();
}
- static Builder builder()
+ boolean contain(Artifact artifact)
{
- return new Builder();
+ Set<String> artifactIds =
groupIdToArtifactIds.get(artifact.getGroupId());
+ return artifactIds.contains(ANY_ARTIFACT_ID) ||
artifactIds.contains(artifact.getArtifactId());
}
static final class Builder
diff --git
a/services/src/test/java/org/apache/druid/cli/PullDependenciesTest.java
b/services/src/test/java/org/apache/druid/cli/PullDependenciesTest.java
index 2d4db23b0e3..851d0f17720 100644
--- a/services/src/test/java/org/apache/druid/cli/PullDependenciesTest.java
+++ b/services/src/test/java/org/apache/druid/cli/PullDependenciesTest.java
@@ -21,16 +21,32 @@ package org.apache.druid.cli;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
-import io.tesla.aether.internal.DefaultTeslaAether;
import org.apache.druid.guice.ExtensionsConfig;
import org.apache.druid.java.util.common.StringUtils;
+import org.apache.maven.repository.internal.MavenRepositorySystemUtils;
+import org.eclipse.aether.DefaultRepositorySystemSession;
+import org.eclipse.aether.RepositorySystem;
+import org.eclipse.aether.RepositorySystemSession;
import org.eclipse.aether.artifact.Artifact;
import org.eclipse.aether.artifact.DefaultArtifact;
+import org.eclipse.aether.connector.basic.BasicRepositoryConnectorFactory;
import org.eclipse.aether.graph.DefaultDependencyNode;
import org.eclipse.aether.graph.Dependency;
-import org.eclipse.aether.graph.DependencyFilter;
import org.eclipse.aether.graph.DependencyNode;
+import org.eclipse.aether.impl.DefaultServiceLocator;
+import org.eclipse.aether.repository.Authentication;
+import org.eclipse.aether.repository.LocalRepository;
+import org.eclipse.aether.repository.Proxy;
+import org.eclipse.aether.repository.RemoteRepository;
+import org.eclipse.aether.resolution.ArtifactRequest;
+import org.eclipse.aether.resolution.ArtifactResult;
import org.eclipse.aether.resolution.DependencyRequest;
+import org.eclipse.aether.resolution.DependencyResult;
+import org.eclipse.aether.spi.connector.RepositoryConnectorFactory;
+import org.eclipse.aether.spi.connector.transport.TransporterFactory;
+import org.eclipse.aether.transport.http.HttpTransporterFactory;
+import org.eclipse.aether.util.artifact.JavaScopes;
+import org.eclipse.aether.util.repository.AuthenticationBuilder;
import org.hamcrest.CoreMatchers;
import org.junit.Assert;
import org.junit.Before;
@@ -50,6 +66,11 @@ import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.spy;
+
public class PullDependenciesTest
{
private static final String EXTENSION_A_COORDINATE =
"groupX:extension_A:123";
@@ -71,23 +92,18 @@ public class PullDependenciesTest
.put(DEPENDENCY_GROUPID,
HADOOP_CLIENT_VULNERABLE_ARTIFACTID1)
.put(DEPENDENCY_GROUPID,
HADOOP_CLIENT_VULNERABLE_ARTIFACTID2)
.build();
-
+ private static File localRepo; // a mock local repository that stores jars
+ private static Map<Artifact, List<String>> extensionToDependency;
@Rule
public final TemporaryFolder temporaryFolder = new TemporaryFolder();
-
- private File localRepo; // a mock local repository that stores jars
-
private final Artifact extension_A = new
DefaultArtifact(EXTENSION_A_COORDINATE);
private final Artifact extension_B = new
DefaultArtifact(EXTENSION_B_COORDINATE);
private final Artifact hadoop_client_2_3_0 = new
DefaultArtifact(HADOOP_CLIENT_2_3_0_COORDINATE);
private final Artifact hadoop_client_2_4_0 = new
DefaultArtifact(HADOOP_CLIENT_2_4_0_COORDINATE);
-
private PullDependencies pullDependencies;
private File rootExtensionsDir;
private File rootHadoopDependenciesDir;
- private Map<Artifact, List<String>> extensionToDependency;
-
@Before
public void setUp() throws Exception
{
@@ -105,18 +121,22 @@ public class PullDependenciesTest
rootExtensionsDir = temporaryFolder.newFolder("extensions");
rootHadoopDependenciesDir =
temporaryFolder.newFolder("druid_hadoop_dependencies");
+ RepositorySystem realRepositorySystem =
RealRepositorySystemUtil.newRepositorySystem();
+ RepositorySystem spyMockRepositorySystem = spy(realRepositorySystem);
+ RepositorySystemSession repositorySystemSession =
RealRepositorySystemUtil.newRepositorySystemSession(
+ spyMockRepositorySystem,
+ localRepo.getPath()
+ );
+
+ doAnswer(invocation -> {
+ DependencyRequest request = invocation.getArgument(1);
+ return
mockDependencyResult(request.getCollectRequest().getRoot().getArtifact());
+
}).when(spyMockRepositorySystem).resolveDependencies(eq(repositorySystemSession),
any(DependencyRequest.class));
+
+
pullDependencies = new PullDependencies(
- new DefaultTeslaAether()
- {
- @Override
- public List<Artifact> resolveArtifacts(DependencyRequest request)
- {
- return getArtifactsForExtension(
- request.getCollectRequest().getRoot().getArtifact(),
- request.getFilter()
- );
- }
- },
+ spyMockRepositorySystem,
+ repositorySystemSession,
new ExtensionsConfig()
{
@Override
@@ -140,14 +160,15 @@ public class PullDependenciesTest
HADOOP_CLIENT_2_4_0_COORDINATE
);
- // Because --clean is specified, pull-deps will first remove existing root
extensions and hadoop dependencies
pullDependencies.clean = true;
}
- private List<Artifact> getArtifactsForExtension(Artifact artifact,
DependencyFilter filter)
+ private DependencyResult mockDependencyResult(Artifact artifact)
{
- final List<String> names = extensionToDependency.get(artifact);
- final List<Artifact> artifacts = new ArrayList<>();
+ final List<String> names = extensionToDependency.getOrDefault(artifact,
Collections.emptyList());
+ final List<ArtifactResult> artifacts = new ArrayList<>();
+ List<DependencyNode> children = new ArrayList<>();
+
for (String name : names) {
final File jarFile = new File(localRepo, name + ".jar");
try {
@@ -156,18 +177,23 @@ public class PullDependenciesTest
catch (IOException e) {
throw new RuntimeException(e);
}
-
- DependencyNode node = new DefaultDependencyNode(
- new Dependency(
- new DefaultArtifact(DEPENDENCY_GROUPID, name, null, "jar",
"1.0", null, jarFile),
- "compile"
- )
+ Artifact depArtifact = new DefaultArtifact("groupid", name, null, "jar",
"1.0",
+ null, jarFile
);
- if (filter.accept(node, Collections.emptyList())) {
- artifacts.add(node.getArtifact());
- }
+ DependencyNode depNode = new DefaultDependencyNode(new
Dependency(depArtifact, JavaScopes.COMPILE));
+ children.add(depNode);
+ ArtifactResult artifactResult = new ArtifactResult(new
ArtifactRequest(depNode));
+ artifactResult.setArtifact(depArtifact);
+ artifacts.add(artifactResult);
}
- return artifacts;
+
+ DependencyNode rootNode = new DefaultDependencyNode(new
Dependency(artifact, JavaScopes.COMPILE));
+ rootNode.setChildren(children);
+
+ DependencyResult result = new DependencyResult(new DependencyRequest());
+ result.setRoot(rootNode);
+ result.setArtifactResults(artifacts);
+ return result;
}
private List<File> getExpectedJarFiles(Artifact artifact)
@@ -299,4 +325,157 @@ public class PullDependenciesTest
Assert.assertThat(dependencies,
CoreMatchers.not(CoreMatchers.hasItem(HADOOP_CLIENT_VULNERABLE_JAR1)));
Assert.assertThat(dependencies,
CoreMatchers.not(CoreMatchers.hasItem(HADOOP_CLIENT_VULNERABLE_JAR2)));
}
+
+ @Test
+ public void testPullDependenciesCleanFlag() throws IOException
+ {
+ File dummyFile1 = new File(rootExtensionsDir, "dummy.txt");
+ File dummyFile2 = new File(rootHadoopDependenciesDir, "dummy.txt");
+ Assert.assertTrue(dummyFile1.createNewFile());
+ Assert.assertTrue(dummyFile2.createNewFile());
+
+ pullDependencies.clean = true;
+ pullDependencies.run();
+
+ Assert.assertFalse(dummyFile1.exists());
+ Assert.assertFalse(dummyFile2.exists());
+ }
+
+ @Test
+ public void testPullDependenciesNoDefaultRemoteRepositories()
+ {
+ pullDependencies.noDefaultRemoteRepositories = true;
+ pullDependencies.remoteRepositories =
ImmutableList.of("https://custom.repo";);
+
+ pullDependencies.run();
+
+ List<RemoteRepository> repositories =
pullDependencies.getRemoteRepositories();
+ Assert.assertEquals(1, repositories.size());
+ Assert.assertEquals("https://custom.repo";, repositories.get(0).getUrl());
+ }
+
+ @Test
+ public void testPullDependenciesDirectoryCreationFailure() throws IOException
+ {
+ if (rootExtensionsDir.exists()) {
+ rootExtensionsDir.delete();
+ }
+ Assert.assertTrue(rootExtensionsDir.createNewFile());
+
+ Assert.assertThrows(IllegalArgumentException.class, () ->
pullDependencies.run());
+ }
+
+ @Test
+ public void testGetArtifactWithValidCoordinate()
+ {
+ String coordinate = "groupX:artifactX:1.0.0";
+ DefaultArtifact artifact = (DefaultArtifact)
pullDependencies.getArtifact(coordinate);
+ Assert.assertEquals("groupX", artifact.getGroupId());
+ Assert.assertEquals("artifactX", artifact.getArtifactId());
+ Assert.assertEquals("1.0.0", artifact.getVersion());
+ }
+
+ @Test
+ public void testGetArtifactwithCoordinateWithoutDefaultVersion()
+ {
+ String coordinate = "groupY:artifactY";
+ Assert.assertThrows(
+ "Bad artifact coordinates groupY:artifactY, expected format is
<groupId>:<artifactId>[:<extension>[:<classifier>]]:<version>",
+ IllegalArgumentException.class,
+ () -> pullDependencies.getArtifact(coordinate)
+ );
+
+ }
+
+ @Test
+ public void testGetArtifactWithCoordinateWithoutVersion()
+ {
+ pullDependencies.defaultVersion = "2.0.0";
+ String coordinate = "groupY:artifactY";
+ DefaultArtifact artifact = (DefaultArtifact)
pullDependencies.getArtifact(coordinate);
+ Assert.assertEquals("groupY", artifact.getGroupId());
+ Assert.assertEquals("artifactY", artifact.getArtifactId());
+ Assert.assertEquals("2.0.0", artifact.getVersion());
+ }
+
+ @Test
+ public void testGetRemoteRepositoriesWithDefaultRepositories()
+ {
+ pullDependencies.noDefaultRemoteRepositories = false; // Use default
remote repositories
+ pullDependencies.remoteRepositories =
ImmutableList.of("https://custom.repo";);
+
+ List<RemoteRepository> repositories =
pullDependencies.getRemoteRepositories();
+ Assert.assertEquals(2, repositories.size());
+ Assert.assertEquals("https://repo1.maven.org/maven2/";,
repositories.get(0).getUrl());
+ Assert.assertEquals("https://custom.repo";, repositories.get(1).getUrl());
+ }
+
+ @Test
+ public void testGetRepositorySystemSessionWithProxyConfiguration()
+ {
+ pullDependencies.useProxy = true;
+ pullDependencies.proxyType = "http";
+ pullDependencies.proxyHost = "localhost";
+ pullDependencies.proxyPort = 8080;
+ pullDependencies.proxyUsername = "user";
+ pullDependencies.proxyPassword = "password";
+
+ DefaultRepositorySystemSession session = (DefaultRepositorySystemSession)
pullDependencies.getRepositorySystemSession();
+
+ LocalRepository localRepo =
session.getLocalRepositoryManager().getRepository();
+ Assert.assertEquals(pullDependencies.localRepository,
localRepo.getBasedir().getAbsolutePath());
+
+ Proxy proxy = session.getProxySelector().getProxy(
+ new RemoteRepository.Builder("test", "default",
"http://example.com";).build()
+ );
+ RemoteRepository testRepository = new RemoteRepository.Builder("test",
"default", "http://example.com";)
+ .setProxy(proxy)
+ .build();
+
+ Assert.assertNotNull(proxy);
+ Assert.assertEquals("localhost", proxy.getHost());
+ Assert.assertEquals(8080, proxy.getPort());
+ Assert.assertEquals("http", proxy.getType());
+
+ Authentication auth = new
AuthenticationBuilder().addUsername("user").addPassword("password").build();
+ Assert.assertEquals(auth, proxy.getAuthentication());
+ }
+
+ @Test
+ public void testGetRepositorySystemSessionWithoutProxyConfiguration()
+ {
+ pullDependencies.useProxy = false;
+ DefaultRepositorySystemSession session = (DefaultRepositorySystemSession)
pullDependencies.getRepositorySystemSession();
+ LocalRepository localRepo =
session.getLocalRepositoryManager().getRepository();
+ Assert.assertEquals(pullDependencies.localRepository,
localRepo.getBasedir().getAbsolutePath());
+ Proxy proxy = session.getProxySelector().getProxy(
+ new RemoteRepository.Builder("test", "default",
"http://example.com";).build()
+ );
+ Assert.assertNull(proxy);
+ }
+
+ private static class RealRepositorySystemUtil
+ {
+ public static RepositorySystem newRepositorySystem()
+ {
+ DefaultServiceLocator locator =
MavenRepositorySystemUtils.newServiceLocator();
+ locator.addService(RepositoryConnectorFactory.class,
BasicRepositoryConnectorFactory.class);
+ locator.addService(TransporterFactory.class,
HttpTransporterFactory.class);
+ return locator.getService(RepositorySystem.class);
+ }
+
+ public static DefaultRepositorySystemSession newRepositorySystemSession(
+ RepositorySystem system,
+ String localRepoPath
+ )
+ {
+ DefaultRepositorySystemSession session =
MavenRepositorySystemUtils.newSession();
+
+ LocalRepository localRepo = new LocalRepository(localRepoPath);
+
session.setLocalRepositoryManager(system.newLocalRepositoryManager(session,
localRepo));
+
+ return session;
+ }
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
