This is an automated email from the ASF dual-hosted git repository. martin_s pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/archiva.git
commit dda90e3302715b65f0d0908e1997e7aaf32b6242 Author: Martin Stockhammer <[email protected]> AuthorDate: Sun Jun 7 11:12:29 2020 +0200 Moving proxy to new repository API --- .../proxy/model/RepositoryProxyHandler.java | 17 +++ .../proxy/DefaultRepositoryProxyHandler.java | 117 +++++++++++++++++++++ .../archiva/repository/LayoutRuntimeException.java | 5 +- .../repository/metadata/base/MetadataTools.java | 49 --------- .../archiva/proxy/CacheFailuresTransferTest.java | 6 +- .../mock/ManagedRepositoryContentMock.java | 14 ++- .../content/ManagedDefaultRepositoryContent.java | 13 --- 7 files changed, 153 insertions(+), 68 deletions(-) diff --git a/archiva-modules/archiva-base/archiva-proxy-api/src/main/java/org/apache/archiva/proxy/model/RepositoryProxyHandler.java b/archiva-modules/archiva-base/archiva-proxy-api/src/main/java/org/apache/archiva/proxy/model/RepositoryProxyHandler.java index dc13b11..f775c39 100644 --- a/archiva-modules/archiva-base/archiva-proxy-api/src/main/java/org/apache/archiva/proxy/model/RepositoryProxyHandler.java +++ b/archiva-modules/archiva-base/archiva-proxy-api/src/main/java/org/apache/archiva/proxy/model/RepositoryProxyHandler.java @@ -24,6 +24,7 @@ import org.apache.archiva.policies.Policy; import org.apache.archiva.policies.ProxyDownloadException; import org.apache.archiva.repository.ManagedRepository; import org.apache.archiva.repository.RepositoryType; +import org.apache.archiva.repository.content.Artifact; import org.apache.archiva.repository.storage.StorageAsset; import java.util.List; @@ -64,6 +65,7 @@ public interface RepositoryProxyHandler * If the artifact is found, it is downloaded and placed into the source repository * filesystem. * + * @deprecated Replaced by {@link #fetchFromProxies(ManagedRepository, Artifact)} * @param repository the source repository to use. (must be a managed repository) * @param artifact the artifact to fetch. * @return the file that was obtained, or null if no content was obtained @@ -73,6 +75,21 @@ public interface RepositoryProxyHandler throws ProxyDownloadException; /** + * Performs the artifact fetch operation against the target repositories + * of the provided source repository. + * <p> + * If the artifact is found, it is downloaded and placed into the source repository + * filesystem. + * + * @param repository the source repository to use. (must be a managed repository) + * @param artifact the artifact to fetch. + * @return the file that was obtained, or null if no content was obtained + * @throws ProxyDownloadException if there was a problem fetching the content from the target repositories. + */ + StorageAsset fetchFromProxies( ManagedRepository repository, Artifact artifact ) + throws ProxyDownloadException; + + /** * Performs the metadata fetch operation against the target repositories * of the provided source repository. * <p> diff --git a/archiva-modules/archiva-base/archiva-proxy/src/main/java/org/apache/archiva/proxy/DefaultRepositoryProxyHandler.java b/archiva-modules/archiva-base/archiva-proxy/src/main/java/org/apache/archiva/proxy/DefaultRepositoryProxyHandler.java index d010581..8cf2e97 100644 --- a/archiva-modules/archiva-base/archiva-proxy/src/main/java/org/apache/archiva/proxy/DefaultRepositoryProxyHandler.java +++ b/archiva-modules/archiva-base/archiva-proxy/src/main/java/org/apache/archiva/proxy/DefaultRepositoryProxyHandler.java @@ -140,7 +140,71 @@ public abstract class DefaultRepositoryProxyHandler implements RepositoryProxyHa return proxyConnectorRuleConfigurations; } + @Override + public StorageAsset fetchFromProxies( ManagedRepository repository, Artifact artifact ) + throws ProxyDownloadException + { + Map<String, Exception> previousExceptions = new LinkedHashMap<>(); + StorageAsset localFile = artifact.getAsset( ); + + Properties requestProperties = new Properties(); + requestProperties.setProperty( "filetype", "artifact" ); + requestProperties.setProperty( "version", artifact.getVersion().getId() ); + requestProperties.setProperty( "managedRepositoryId", repository.getId() ); + + List<ProxyConnector> connectors = getProxyConnectors( repository ); + for ( ProxyConnector connector : connectors ) + { + if ( !connector.isEnabled() ) + { + continue; + } + RemoteRepository targetRepository = connector.getTargetRepository(); + requestProperties.setProperty( "remoteRepositoryId", targetRepository.getId() ); + + StorageAsset targetFile = targetRepository.getAsset( localFile.getPath( ) ); + // Removing the leading '/' from the path + String targetPath = targetFile.getPath( ).substring( 1 ); + try + { + StorageAsset downloadedFile = + transferFile( connector, targetRepository, targetPath, repository, localFile, requestProperties, + true ); + + if ( fileExists(downloadedFile) ) + { + log.debug( "Successfully transferred: {}", downloadedFile.getPath() ); + return downloadedFile; + } + } + catch ( NotFoundException e ) + { + log.debug( "Artifact {} not found on repository \"{}\".", artifact.getId(), + targetRepository.getId() ); + } + catch ( NotModifiedException e ) + { + log.debug( "Artifact {} not updated on repository \"{}\".", artifact.getId(), + targetRepository.getId() ); + } + catch ( ProxyException e ) + { + validatePolicies( this.downloadErrorPolicies, connector.getPolicies(), requestProperties, artifact, + targetRepository.getContent(), localFile, e, previousExceptions ); + } + } + + if ( !previousExceptions.isEmpty() ) + { + throw new ProxyDownloadException( "Failures occurred downloading from some remote repositories", + previousExceptions ); + } + + log.debug( "Exhausted all target repositories, artifact {} not found.", artifact.getId() ); + + return null; + } @Override public StorageAsset fetchFromProxies( ManagedRepository repository, ArtifactReference artifact ) @@ -672,6 +736,59 @@ public abstract class DefaultRepositoryProxyHandler implements RepositoryProxyHa } private void validatePolicies( Map<String, DownloadErrorPolicy> policies, Map<Policy, PolicyOption> settings, + Properties request, Artifact artifact, RemoteRepositoryContent content, + StorageAsset localFile, Exception exception, Map<String, Exception> previousExceptions ) + throws ProxyDownloadException + { + boolean process = true; + for ( Map.Entry<String, ? extends DownloadErrorPolicy> entry : policies.entrySet() ) + { + + // olamy with spring rolehint is now downloadPolicy#hint + // so substring after last # to get the hint as with plexus + String key = entry.getValue( ).getId( ); + DownloadErrorPolicy policy = entry.getValue(); + PolicyOption option = settings.containsKey( policy ) ? settings.get(policy) : policy.getDefaultOption(); + + log.debug( "Applying [{}] policy with [{}]", key, option ); + try + { + // all policies must approve the exception, any can cancel + process = policy.applyPolicy( option, request, localFile, exception, previousExceptions ); + if ( !process ) + { + break; + } + } + catch ( PolicyConfigurationException e ) + { + log.error( e.getMessage(), e ); + } + } + + if ( process ) + { + // if the exception was queued, don't throw it + if ( !previousExceptions.containsKey( content.getId() ) ) + { + throw new ProxyDownloadException( + "An error occurred in downloading from the remote repository, and the policy is to fail immediately", + content.getId(), exception ); + } + } + else + { + // if the exception was queued, but cancelled, remove it + previousExceptions.remove( content.getId() ); + } + + log.warn( + "Transfer error from repository {} for artifact {} , continuing to next repository. Error message: {}", + content.getRepository().getId(), artifact.getId(), exception.getMessage() ); + log.debug( "Full stack trace", exception ); + } + + private void validatePolicies( Map<String, DownloadErrorPolicy> policies, Map<Policy, PolicyOption> settings, Properties request, ArtifactReference artifact, RemoteRepositoryContent content, StorageAsset localFile, Exception exception, Map<String, Exception> previousExceptions ) throws ProxyDownloadException diff --git a/archiva-modules/archiva-base/archiva-repository-api/src/main/java/org/apache/archiva/repository/LayoutRuntimeException.java b/archiva-modules/archiva-base/archiva-repository-api/src/main/java/org/apache/archiva/repository/LayoutRuntimeException.java index 9abc5e0..61477b5 100644 --- a/archiva-modules/archiva-base/archiva-repository-api/src/main/java/org/apache/archiva/repository/LayoutRuntimeException.java +++ b/archiva-modules/archiva-base/archiva-repository-api/src/main/java/org/apache/archiva/repository/LayoutRuntimeException.java @@ -19,11 +19,10 @@ package org.apache.archiva.repository; * under the License. */ -import org.apache.archiva.common.ArchivaException; - /** - * LayoutException + * Layout exception for methods used in lambda expressions. * + * @author Martin Stockhammer <[email protected]> * */ public class LayoutRuntimeException extends RuntimeException diff --git a/archiva-modules/archiva-base/archiva-repository-layer/src/main/java/org/apache/archiva/repository/metadata/base/MetadataTools.java b/archiva-modules/archiva-base/archiva-repository-layer/src/main/java/org/apache/archiva/repository/metadata/base/MetadataTools.java index ce82fa1..379179c 100644 --- a/archiva-modules/archiva-base/archiva-repository-layer/src/main/java/org/apache/archiva/repository/metadata/base/MetadataTools.java +++ b/archiva-modules/archiva-base/archiva-repository-layer/src/main/java/org/apache/archiva/repository/metadata/base/MetadataTools.java @@ -913,55 +913,6 @@ public class MetadataTools } } - /** - * Get the first Artifact found in the provided VersionedReference location. - * - * @param managedRepository the repository to search within. - * @param reference the reference to the versioned reference to search within - * @return the ArtifactReference to the first artifact located within the versioned reference. or null if - * no artifact was found within the versioned reference. - * @throws IOException if the versioned reference is invalid (example: doesn't exist, or isn't a directory) - * @throws LayoutException - */ - public ArtifactReference getFirstArtifact( BaseRepositoryContentLayout managedRepository, - VersionedReference reference ) - throws LayoutException, IOException - { - String path = toPath( reference ); - - int idx = path.lastIndexOf( '/' ); - if ( idx > 0 ) - { - path = path.substring( 0, idx ); - } - - StorageAsset repoDir = managedRepository.getGenericContent( ).getRepository( ).getRoot(); - - if ( !repoDir.exists()) - { - throw new IOException( "Unable to gather the list of snapshot versions on a non-existant directory: " - + repoDir.toString() ); - } - - if ( !repoDir.isContainer()) - { - throw new IOException( - "Unable to gather the list of snapshot versions on a non-directory: " + repoDir.toString() ); - } - - Path repoRoot = repoDir.getFilePath( ); - try(Stream<Path> stream = Files.list(repoRoot)) { - String result = stream.filter( Files::isRegularFile ).map( path1 -> - repoRoot.relativize( path1 ).toString() - ).filter( filetypes::matchesArtifactPattern ).findFirst().orElse( null ); - if (result!=null) { - return managedRepository.getGenericContent().toArtifactReference( result ); - } - } - // No artifact was found. - return null; - } - public ArchivaConfiguration getConfiguration() { return configuration; diff --git a/archiva-modules/archiva-maven/archiva-maven-proxy/src/test/java/org/apache/archiva/proxy/CacheFailuresTransferTest.java b/archiva-modules/archiva-maven/archiva-maven-proxy/src/test/java/org/apache/archiva/proxy/CacheFailuresTransferTest.java index 5afb561..4f60193 100644 --- a/archiva-modules/archiva-maven/archiva-maven-proxy/src/test/java/org/apache/archiva/proxy/CacheFailuresTransferTest.java +++ b/archiva-modules/archiva-maven/archiva-maven-proxy/src/test/java/org/apache/archiva/proxy/CacheFailuresTransferTest.java @@ -27,6 +27,7 @@ import org.apache.archiva.policies.ReleasesPolicy; import org.apache.archiva.policies.SnapshotsPolicy; import org.apache.archiva.policies.urlcache.UrlFailureCache; import org.apache.archiva.repository.BaseRepositoryContentLayout; +import org.apache.archiva.repository.content.Artifact; import org.apache.archiva.repository.storage.StorageAsset; import org.apache.maven.wagon.ResourceDoesNotExistException; import org.easymock.EasyMock; @@ -64,7 +65,8 @@ public class CacheFailuresTransferTest assertNotExistsInManagedDefaultRepo( expectedFile ); BaseRepositoryContentLayout layout = managedDefaultRepository.getLayout( BaseRepositoryContentLayout.class ); - ArtifactReference artifact = managedDefaultRepository.toArtifactReference( path ); + Artifact artifact = layout.getArtifact( path ); + // ArtifactReference artifact = managedDefaultRepository.toArtifactReference( path ); // Configure Repository (usually done within archiva.xml configuration) saveRemoteRepositoryConfig( "badproxied1", "Bad Proxied 1", "http://bad.machine.com/repo/";, "default" ); @@ -109,7 +111,7 @@ public class CacheFailuresTransferTest assertNotExistsInManagedDefaultRepo( expectedFile ); BaseRepositoryContentLayout layout = managedDefaultRepository.getLayout( BaseRepositoryContentLayout.class ); - ArtifactReference artifact = managedDefaultRepository.toArtifactReference( path ); + Artifact artifact = layout.getArtifact( path ); // Configure Repository (usually done within archiva.xml configuration) saveRemoteRepositoryConfig( "badproxied1", "Bad Proxied 1", "http://bad.machine.com/repo/";, "default" ); diff --git a/archiva-modules/archiva-maven/archiva-maven-proxy/src/test/java/org/apache/archiva/repository/mock/ManagedRepositoryContentMock.java b/archiva-modules/archiva-maven/archiva-maven-proxy/src/test/java/org/apache/archiva/repository/mock/ManagedRepositoryContentMock.java index c59bfad..7048470 100644 --- a/archiva-modules/archiva-maven/archiva-maven-proxy/src/test/java/org/apache/archiva/repository/mock/ManagedRepositoryContentMock.java +++ b/archiva-modules/archiva-maven/archiva-maven-proxy/src/test/java/org/apache/archiva/repository/mock/ManagedRepositoryContentMock.java @@ -174,7 +174,19 @@ public class ManagedRepositoryContentMock implements BaseRepositoryContentLayout @Override public Artifact getArtifact( String path ) throws LayoutException, ContentAccessException { - return null; + StorageAsset asset = fsStorage.getAsset( path.toString( ) ); + String artifactId = asset.getName( ); + StorageAsset namespacePath = asset.getParent( ).getParent( ).getParent( ); + String namespaceId = namespacePath.getPath( ).replace( "/", "." ); + StorageAsset projectPath = asset.getParent( ).getParent( ); + String projectId = projectPath.getName( ); + StorageAsset versionPath = asset.getParent( ); + String versionId = versionPath.getName( ); + ArchivaNamespace ns = ArchivaNamespace.withRepository( repository.getContent( ) ).withAsset( namespacePath ).withNamespace( namespaceId ).build( ); + ArchivaProject project = ArchivaProject.withRepository( repository.getContent( ) ).withAsset( projectPath ).withNamespace( ns ).withId( projectId ).build( ); + ArchivaVersion version = ArchivaVersion.withRepository( repository.getContent( ) ).withAsset( versionPath ).withProject( project ).withVersion( versionId ).build( ); + ArchivaArtifact artifact = ArchivaArtifact.withAsset( asset ).withVersion( version ).withId( projectId ).withArtifactVersion( versionId ).build( ); + return artifact; } @Override diff --git a/archiva-modules/archiva-maven/archiva-maven-repository/src/main/java/org/apache/archiva/repository/maven/content/ManagedDefaultRepositoryContent.java b/archiva-modules/archiva-maven/archiva-maven-repository/src/main/java/org/apache/archiva/repository/maven/content/ManagedDefaultRepositoryContent.java index 319eaa7..eedd366 100644 --- a/archiva-modules/archiva-maven/archiva-maven-repository/src/main/java/org/apache/archiva/repository/maven/content/ManagedDefaultRepositoryContent.java +++ b/archiva-modules/archiva-maven/archiva-maven-repository/src/main/java/org/apache/archiva/repository/maven/content/ManagedDefaultRepositoryContent.java @@ -1627,19 +1627,6 @@ public class ManagedDefaultRepositoryContent } - // The variant with runtime exception for stream usage - private ArtifactReference toArtifactRef( String path ) - { - try - { - return toArtifactReference( path ); - } - catch ( LayoutException e ) - { - throw new RuntimeException( e ); - } - } - public void setFiletypes( FileTypes filetypes ) { this.filetypes = filetypes;
