This is an automated email from the ASF dual-hosted git repository. martin_s pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/archiva.git
commit 46fd585f40f0264b8889154e1924eb8df991b0f3 Author: Martin Stockhammer <[email protected]> AuthorDate: Thu Jun 11 14:33:00 2020 +0200 Improving repository API --- .../apache/archiva/common/utils/VersionUtil.java | 2 + .../core/repository/DaysOldRepositoryPurge.java | 140 ++++++++++----------- .../repository/RetentionCountRepositoryPurge.java | 104 ++++++++------- .../proxy/model/RepositoryProxyHandler.java | 16 +++ .../proxy/DefaultRepositoryProxyHandler.java | 72 ++++++++++- .../archiva/repository/RepositoryRequestInfo.java | 10 ++ .../content/ManagedDefaultRepositoryContent.java | 112 ++++++++++++----- .../maven/content/MavenRepositoryRequestInfo.java | 19 ++- .../metadata/storage/Maven2RepositoryStorage.java | 92 ++++++++++++++ .../ManagedDefaultRepositoryContentTest.java | 2 +- .../content/MavenRepositoryRequestInfoTest.java | 9 +- .../archiva/security/mock/MockBeanServices.java | 7 ++ .../archiva/webdav/ArchivaDavResourceFactory.java | 16 +-- .../repository/storage/RepositoryStorage.java | 16 +++ .../cassandra/MockRepositoryStorage.java | 7 ++ 15 files changed, 458 insertions(+), 166 deletions(-) diff --git a/archiva-modules/archiva-base/archiva-common/src/main/java/org/apache/archiva/common/utils/VersionUtil.java b/archiva-modules/archiva-base/archiva-common/src/main/java/org/apache/archiva/common/utils/VersionUtil.java index c1a3ac6..408d7b4 100644 --- a/archiva-modules/archiva-base/archiva-common/src/main/java/org/apache/archiva/common/utils/VersionUtil.java +++ b/archiva-modules/archiva-base/archiva-common/src/main/java/org/apache/archiva/common/utils/VersionUtil.java @@ -194,4 +194,6 @@ public class VersionUtil } return version; } + + } diff --git a/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/archiva/consumers/core/repository/DaysOldRepositoryPurge.java b/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/archiva/consumers/core/repository/DaysOldRepositoryPurge.java index fb42390..282f45c 100644 --- a/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/archiva/consumers/core/repository/DaysOldRepositoryPurge.java +++ b/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/archiva/consumers/core/repository/DaysOldRepositoryPurge.java @@ -23,9 +23,9 @@ import org.apache.archiva.common.utils.VersionComparator; import org.apache.archiva.common.utils.VersionUtil; import org.apache.archiva.metadata.audit.RepositoryListener; import org.apache.archiva.metadata.repository.RepositorySession; -import org.apache.archiva.repository.ManagedRepositoryContent; -import org.apache.archiva.repository.LayoutException; import org.apache.archiva.repository.BaseRepositoryContentLayout; +import org.apache.archiva.repository.LayoutException; +import org.apache.archiva.repository.ManagedRepositoryContent; import org.apache.archiva.repository.content.Artifact; import org.apache.archiva.repository.content.ContentItem; import org.apache.archiva.repository.content.base.ArchivaItemSelector; @@ -74,94 +74,94 @@ public class DaysOldRepositoryPurge { ContentItem item = repository.toItem( path ); - if ( item instanceof Artifact ) + Artifact artifactItem = repository.getLayout( BaseRepositoryContentLayout.class ).adaptItem( Artifact.class, item ); + + if ( !artifactItem.exists( ) ) { - Artifact artifactItem = (Artifact) item; + return; + } - if ( !artifactItem.exists( ) ) - { - return; - } + // ArtifactReference artifact = repository.toArtifactReference( path ); - // ArtifactReference artifact = repository.toArtifactReference( path ); - - Calendar olderThanThisDate = Calendar.getInstance( TimeZone.getTimeZone( "UTC" ) ); - olderThanThisDate.add( Calendar.DATE, -retentionPeriod ); - - ArchivaItemSelector selector = ArchivaItemSelector.builder( ) - .withNamespace( artifactItem.getVersion( ).getProject( ).getNamespace( ).getId( ) ) - .withProjectId( artifactItem.getVersion( ).getProject( ).getId( ) ) - .withVersion( artifactItem.getVersion( ).getId( ) ) - .withClassifier( "*" ) - .includeRelatedArtifacts( ) - .build( ); - - List<String> artifactVersions; - try( Stream<? extends Artifact> stream = repository.getLayout( BaseRepositoryContentLayout.class ).newArtifactStream( selector )){ - artifactVersions = stream.map( a -> a.getArtifactVersion( ) ) - .filter( StringUtils::isNotEmpty ) - .distinct() - .collect( Collectors.toList( ) ); - } + Calendar olderThanThisDate = Calendar.getInstance( TimeZone.getTimeZone( "UTC" ) ); + olderThanThisDate.add( Calendar.DATE, -retentionPeriod ); - Collections.sort( artifactVersions, VersionComparator.getInstance( ) ); + ArchivaItemSelector selector = ArchivaItemSelector.builder( ) + .withNamespace( artifactItem.getVersion( ).getProject( ).getNamespace( ).getId( ) ) + .withProjectId( artifactItem.getVersion( ).getProject( ).getId( ) ) + .withVersion( artifactItem.getVersion( ).getId( ) ) + .withClassifier( "*" ) + .includeRelatedArtifacts( ) + .build( ); - if ( retentionCount > artifactVersions.size( ) ) - { - // Done. nothing to do here. skip it. - return; - } + List<String> artifactVersions; + try ( Stream<? extends Artifact> stream = repository.getLayout( BaseRepositoryContentLayout.class ).newArtifactStream( selector ) ) + { + artifactVersions = stream.map( a -> a.getArtifactVersion( ) ) + .filter( StringUtils::isNotEmpty ) + .distinct( ) + .collect( Collectors.toList( ) ); + } + + Collections.sort( artifactVersions, VersionComparator.getInstance( ) ); - int countToPurge = artifactVersions.size( ) - retentionCount; + if ( retentionCount > artifactVersions.size( ) ) + { + // Done. nothing to do here. skip it. + return; + } + int countToPurge = artifactVersions.size( ) - retentionCount; - ArchivaItemSelector.Builder artifactSelectorBuilder = ArchivaItemSelector.builder( ) - .withNamespace( artifactItem.getVersion( ).getProject( ).getNamespace( ).getId( ) ) - .withProjectId( artifactItem.getVersion( ).getProject( ).getId( ) ) - .withVersion( artifactItem.getVersion( ).getId( ) ) - .withArtifactId( artifactItem.getId() ) - .withClassifier( "*" ) - .includeRelatedArtifacts( ); - Set<Artifact> artifactsToDelete = new HashSet<>( ); - for ( String version : artifactVersions ) + ArchivaItemSelector.Builder artifactSelectorBuilder = ArchivaItemSelector.builder( ) + .withNamespace( artifactItem.getVersion( ).getProject( ).getNamespace( ).getId( ) ) + .withProjectId( artifactItem.getVersion( ).getProject( ).getId( ) ) + .withVersion( artifactItem.getVersion( ).getId( ) ) + .withArtifactId( artifactItem.getId( ) ) + .withClassifier( "*" ) + .includeRelatedArtifacts( ); + + Set<Artifact> artifactsToDelete = new HashSet<>( ); + for ( String version : artifactVersions ) + { + if ( countToPurge-- <= 0 ) { - if ( countToPurge-- <= 0 ) - { - break; - } + break; + } - ArchivaItemSelector artifactSelector = artifactSelectorBuilder.withArtifactVersion( version ).build( ); - try - { + ArchivaItemSelector artifactSelector = artifactSelectorBuilder.withArtifactVersion( version ).build( ); + try + { - // Is this a generic snapshot "1.0-SNAPSHOT" ? - if ( VersionUtil.isGenericSnapshot( version ) ) + // Is this a generic snapshot "1.0-SNAPSHOT" ? + if ( VersionUtil.isGenericSnapshot( version ) ) + { + List<? extends Artifact> artifactList = repository.getLayout( BaseRepositoryContentLayout.class ).getArtifacts( artifactSelector ); + if ( artifactList.size( ) > 0 && artifactList.get( 0 ).getAsset( ).getModificationTime( ).toEpochMilli( ) < olderThanThisDate.getTimeInMillis( ) ) { - List<? extends Artifact> artifactList = repository.getLayout( BaseRepositoryContentLayout.class ).getArtifacts( artifactSelector ); - if ( artifactList.size()>0 && artifactList.get(0).getAsset().getModificationTime( ).toEpochMilli( ) < olderThanThisDate.getTimeInMillis( ) ) - { - artifactsToDelete.addAll( artifactList ); - } + artifactsToDelete.addAll( artifactList ); } - // Is this a timestamp snapshot "1.0-20070822.123456-42" ? - else if ( VersionUtil.isUniqueSnapshot( version ) ) - { - Calendar timestampCal = uniqueSnapshotToCalendar( version ); + } + // Is this a timestamp snapshot "1.0-20070822.123456-42" ? + else if ( VersionUtil.isUniqueSnapshot( version ) ) + { + Calendar timestampCal = uniqueSnapshotToCalendar( version ); - if ( timestampCal.getTimeInMillis( ) < olderThanThisDate.getTimeInMillis( ) ) - { - artifactsToDelete.addAll( repository.getLayout( BaseRepositoryContentLayout.class ).getArtifacts( artifactSelector ) ); - } + if ( timestampCal.getTimeInMillis( ) < olderThanThisDate.getTimeInMillis( ) ) + { + artifactsToDelete.addAll( repository.getLayout( BaseRepositoryContentLayout.class ).getArtifacts( artifactSelector ) ); } - } catch ( IllegalArgumentException e ) { - log.error( "Bad selector for artifact: {}", e.getMessage( ), e ); - // continue } } - purge( artifactsToDelete ); + catch ( IllegalArgumentException e ) + { + log.error( "Bad selector for artifact: {}", e.getMessage( ), e ); + // continue + } } + purge( artifactsToDelete ); } catch ( LayoutException e ) { diff --git a/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/archiva/consumers/core/repository/RetentionCountRepositoryPurge.java b/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/archiva/consumers/core/repository/RetentionCountRepositoryPurge.java index ec3a242..74df6a6 100644 --- a/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/archiva/consumers/core/repository/RetentionCountRepositoryPurge.java +++ b/archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/archiva/consumers/core/repository/RetentionCountRepositoryPurge.java @@ -21,12 +21,12 @@ package org.apache.archiva.consumers.core.repository; import org.apache.archiva.common.utils.VersionComparator; import org.apache.archiva.common.utils.VersionUtil; +import org.apache.archiva.metadata.audit.RepositoryListener; import org.apache.archiva.metadata.repository.RepositorySession; import org.apache.archiva.model.ArtifactReference; -import org.apache.archiva.repository.ManagedRepositoryContent; -import org.apache.archiva.repository.LayoutException; import org.apache.archiva.repository.BaseRepositoryContentLayout; -import org.apache.archiva.metadata.audit.RepositoryListener; +import org.apache.archiva.repository.LayoutException; +import org.apache.archiva.repository.ManagedRepositoryContent; import org.apache.archiva.repository.content.Artifact; import org.apache.archiva.repository.content.ContentItem; import org.apache.archiva.repository.content.base.ArchivaItemSelector; @@ -61,67 +61,65 @@ public class RetentionCountRepositoryPurge try { ContentItem item = repository.toItem( path ); - if (item instanceof Artifact ) + BaseRepositoryContentLayout layout = repository.getLayout( BaseRepositoryContentLayout.class ); + Artifact artifact = layout.adaptItem( Artifact.class, item ); + if ( !artifact.exists( ) ) { - Artifact artifact = (Artifact) item; - if (!artifact.exists()) { - return; - } + return; + } - if ( VersionUtil.isSnapshot( artifact.getVersion( ).getId() ) ) - { - ArchivaItemSelector selector = ArchivaItemSelector.builder( ) - .withNamespace( artifact.getVersion( ).getProject( ).getNamespace( ).getId( ) ) - .withProjectId( artifact.getVersion( ).getProject( ).getId( ) ) - .withArtifactId( artifact.getId( ) ) - .withVersion( artifact.getVersion( ).getId( ) ) - .withClassifier( "*" ) - .includeRelatedArtifacts() - .build( ); + if ( VersionUtil.isSnapshot( artifact.getVersion( ).getId( ) ) ) + { + ArchivaItemSelector selector = ArchivaItemSelector.builder( ) + .withNamespace( artifact.getVersion( ).getProject( ).getNamespace( ).getId( ) ) + .withProjectId( artifact.getVersion( ).getProject( ).getId( ) ) + .withArtifactId( artifact.getId( ) ) + .withVersion( artifact.getVersion( ).getId( ) ) + .withClassifier( "*" ) + .includeRelatedArtifacts( ) + .build( ); - List<String> versions; - try( Stream<? extends Artifact> stream = repository.getLayout( BaseRepositoryContentLayout.class ).newArtifactStream( selector) ){ - versions = stream.map( a -> a.getArtifactVersion( ) ) - .filter( StringUtils::isNotEmpty ) - .distinct() - .collect( Collectors.toList( ) ); - } + List<String> versions; + try ( Stream<? extends Artifact> stream = repository.getLayout( BaseRepositoryContentLayout.class ).newArtifactStream( selector ) ) + { + versions = stream.map( a -> a.getArtifactVersion( ) ) + .filter( StringUtils::isNotEmpty ) + .distinct( ) + .collect( Collectors.toList( ) ); + } + + Collections.sort( versions, VersionComparator.getInstance( ) ); - Collections.sort( versions, VersionComparator.getInstance( ) ); + if ( retentionCount > versions.size( ) ) + { + log.trace( "No deletion, because retention count is higher than actual number of artifacts." ); + // Done. nothing to do here. skip it. + return; + } - if ( retentionCount > versions.size( ) ) + ArchivaItemSelector.Builder selectorBuilder = ArchivaItemSelector.builder( ) + .withNamespace( artifact.getVersion( ).getProject( ).getNamespace( ).getId( ) ) + .withProjectId( artifact.getVersion( ).getProject( ).getId( ) ) + .withArtifactId( artifact.getId( ) ) + .withClassifier( "*" ) + .includeRelatedArtifacts( ) + .withVersion( artifact.getVersion( ).getId( ) ); + int countToPurge = versions.size( ) - retentionCount; + Set<Artifact> artifactsToDelete = new HashSet<>( ); + for ( String version : versions ) + { + if ( countToPurge-- <= 0 ) { - log.trace( "No deletion, because retention count is higher than actual number of artifacts." ); - // Done. nothing to do here. skip it. - return; + break; } - - ArchivaItemSelector.Builder selectorBuilder = ArchivaItemSelector.builder( ) - .withNamespace( artifact.getVersion( ).getProject( ).getNamespace( ).getId( ) ) - .withProjectId( artifact.getVersion( ).getProject( ).getId( ) ) - .withArtifactId( artifact.getId( ) ) - .withClassifier( "*" ) - .includeRelatedArtifacts() - .withVersion( artifact.getVersion( ).getId( ) ); - int countToPurge = versions.size( ) - retentionCount; - Set<Artifact> artifactsToDelete = new HashSet<>( ); - for ( String version : versions ) + List<? extends Artifact> delArtifacts = repository.getLayout( BaseRepositoryContentLayout.class ).getArtifacts( selectorBuilder.withArtifactVersion( version ).build( ) ); + if ( delArtifacts != null && delArtifacts.size( ) > 0 ) { - if ( countToPurge-- <= 0 ) - { - break; - } - List<? extends Artifact> delArtifacts = repository.getLayout( BaseRepositoryContentLayout.class ).getArtifacts( selectorBuilder.withArtifactVersion( version ).build( ) ); - if (delArtifacts!=null && delArtifacts.size()>0) - { - artifactsToDelete.addAll( delArtifacts ); - } + artifactsToDelete.addAll( delArtifacts ); } - purge( artifactsToDelete ); } - } else { - throw new RepositoryPurgeException( "Bad artifact path " + path ); + purge( artifactsToDelete ); } } catch ( LayoutException le ) diff --git a/archiva-modules/archiva-base/archiva-proxy-api/src/main/java/org/apache/archiva/proxy/model/RepositoryProxyHandler.java b/archiva-modules/archiva-base/archiva-proxy-api/src/main/java/org/apache/archiva/proxy/model/RepositoryProxyHandler.java index f775c39..f6c9259 100644 --- a/archiva-modules/archiva-base/archiva-proxy-api/src/main/java/org/apache/archiva/proxy/model/RepositoryProxyHandler.java +++ b/archiva-modules/archiva-base/archiva-proxy-api/src/main/java/org/apache/archiva/proxy/model/RepositoryProxyHandler.java @@ -25,6 +25,7 @@ import org.apache.archiva.policies.ProxyDownloadException; import org.apache.archiva.repository.ManagedRepository; import org.apache.archiva.repository.RepositoryType; import org.apache.archiva.repository.content.Artifact; +import org.apache.archiva.repository.content.ItemSelector; import org.apache.archiva.repository.storage.StorageAsset; import java.util.List; @@ -90,6 +91,21 @@ public interface RepositoryProxyHandler throws ProxyDownloadException; /** + * Performs the artifact fetch operation against the target repositories + * of the provided source repository. + * <p> + * If the artifact is found, it is downloaded and placed into the source repository + * filesystem. + * + * @param repository the source repository to use. (must be a managed repository) + * @param artifactSelector the artifact to fetch. + * @return the file that was obtained, or null if no content was obtained + * @throws ProxyDownloadException if there was a problem fetching the content from the target repositories. + */ + StorageAsset fetchFromProxies( ManagedRepository repository, ItemSelector artifactSelector ) + throws ProxyDownloadException; + + /** * Performs the metadata fetch operation against the target repositories * of the provided source repository. * <p> diff --git a/archiva-modules/archiva-base/archiva-proxy/src/main/java/org/apache/archiva/proxy/DefaultRepositoryProxyHandler.java b/archiva-modules/archiva-base/archiva-proxy/src/main/java/org/apache/archiva/proxy/DefaultRepositoryProxyHandler.java index 8cf2e97..2373fb9 100644 --- a/archiva-modules/archiva-base/archiva-proxy/src/main/java/org/apache/archiva/proxy/DefaultRepositoryProxyHandler.java +++ b/archiva-modules/archiva-base/archiva-proxy/src/main/java/org/apache/archiva/proxy/DefaultRepositoryProxyHandler.java @@ -50,6 +50,7 @@ import org.apache.archiva.repository.RemoteRepository; import org.apache.archiva.repository.RemoteRepositoryContent; import org.apache.archiva.repository.RepositoryType; import org.apache.archiva.repository.content.Artifact; +import org.apache.archiva.repository.content.ContentItem; import org.apache.archiva.repository.content.ItemSelector; import org.apache.archiva.repository.content.base.ArchivaItemSelector; import org.apache.archiva.repository.metadata.base.MetadataTools; @@ -207,6 +208,73 @@ public abstract class DefaultRepositoryProxyHandler implements RepositoryProxyHa } @Override + public StorageAsset fetchFromProxies( ManagedRepository repository, ItemSelector artifactSelector ) + throws ProxyDownloadException + { + Map<String, Exception> previousExceptions = new LinkedHashMap<>(); + ContentItem item = repository.getContent( ).getItem( artifactSelector ); + StorageAsset localFile = item.getAsset( ); + + Properties requestProperties = new Properties(); + requestProperties.setProperty( "filetype", "artifact" ); + requestProperties.setProperty( "version", artifactSelector.getVersion() ); + requestProperties.setProperty( "managedRepositoryId", repository.getId() ); + + List<ProxyConnector> connectors = getProxyConnectors( repository ); + for ( ProxyConnector connector : connectors ) + { + if ( !connector.isEnabled() ) + { + continue; + } + + RemoteRepository targetRepository = connector.getTargetRepository(); + requestProperties.setProperty( "remoteRepositoryId", targetRepository.getId() ); + + StorageAsset targetFile = targetRepository.getAsset( localFile.getPath( ) ); + // Removing the leading '/' from the path + String targetPath = targetFile.getPath( ).substring( 1 ); + try + { + StorageAsset downloadedFile = + transferFile( connector, targetRepository, targetPath, repository, localFile, requestProperties, + true ); + + if ( fileExists(downloadedFile) ) + { + log.debug( "Successfully transferred: {}", downloadedFile.getPath() ); + return downloadedFile; + } + } + catch ( NotFoundException e ) + { + log.debug( "Artifact {} not found on repository \"{}\".", item, + targetRepository.getId() ); + } + catch ( NotModifiedException e ) + { + log.debug( "Artifact {} not updated on repository \"{}\".", item, + targetRepository.getId() ); + } + catch ( ProxyException e ) + { + validatePolicies( this.downloadErrorPolicies, connector.getPolicies(), requestProperties, item, + targetRepository.getContent(), localFile, e, previousExceptions ); + } + } + + if ( !previousExceptions.isEmpty() ) + { + throw new ProxyDownloadException( "Failures occurred downloading from some remote repositories", + previousExceptions ); + } + + log.debug( "Exhausted all target repositories, artifact {} not found.", item ); + + return null; + } + + @Override public StorageAsset fetchFromProxies( ManagedRepository repository, ArtifactReference artifact ) throws ProxyDownloadException { @@ -736,7 +804,7 @@ public abstract class DefaultRepositoryProxyHandler implements RepositoryProxyHa } private void validatePolicies( Map<String, DownloadErrorPolicy> policies, Map<Policy, PolicyOption> settings, - Properties request, Artifact artifact, RemoteRepositoryContent content, + Properties request, ContentItem artifact, RemoteRepositoryContent content, StorageAsset localFile, Exception exception, Map<String, Exception> previousExceptions ) throws ProxyDownloadException { @@ -784,7 +852,7 @@ public abstract class DefaultRepositoryProxyHandler implements RepositoryProxyHa log.warn( "Transfer error from repository {} for artifact {} , continuing to next repository. Error message: {}", - content.getRepository().getId(), artifact.getId(), exception.getMessage() ); + content.getRepository().getId(), artifact, exception.getMessage() ); log.debug( "Full stack trace", exception ); } diff --git a/archiva-modules/archiva-base/archiva-repository-api/src/main/java/org/apache/archiva/repository/RepositoryRequestInfo.java b/archiva-modules/archiva-base/archiva-repository-api/src/main/java/org/apache/archiva/repository/RepositoryRequestInfo.java index 04700e7..52cfb2c 100644 --- a/archiva-modules/archiva-base/archiva-repository-api/src/main/java/org/apache/archiva/repository/RepositoryRequestInfo.java +++ b/archiva-modules/archiva-base/archiva-repository-api/src/main/java/org/apache/archiva/repository/RepositoryRequestInfo.java @@ -20,6 +20,7 @@ package org.apache.archiva.repository; */ import org.apache.archiva.model.ArtifactReference; +import org.apache.archiva.repository.content.ItemSelector; import org.apache.archiva.repository.features.RepositoryFeature; /** @@ -42,6 +43,15 @@ public interface RepositoryRequestInfo */ ArtifactReference toArtifactReference( String requestPath ) throws LayoutException; + + /** + * Returns the item selector that matches the given path. + * @param requestPath the request path which may be different from the filesystem structure + * @return the item selector + * @throws LayoutException if the path is not valid for the given repository layout + */ + ItemSelector toItemSelector( String requestPath ) throws LayoutException; + /** * <p> * Tests the path to see if it conforms to the expectations of a metadata request. diff --git a/archiva-modules/archiva-maven/archiva-maven-repository/src/main/java/org/apache/archiva/repository/maven/content/ManagedDefaultRepositoryContent.java b/archiva-modules/archiva-maven/archiva-maven-repository/src/main/java/org/apache/archiva/repository/maven/content/ManagedDefaultRepositoryContent.java index 7341747..b66207d 100644 --- a/archiva-modules/archiva-maven/archiva-maven-repository/src/main/java/org/apache/archiva/repository/maven/content/ManagedDefaultRepositoryContent.java +++ b/archiva-modules/archiva-maven/archiva-maven-repository/src/main/java/org/apache/archiva/repository/maven/content/ManagedDefaultRepositoryContent.java @@ -20,6 +20,7 @@ package org.apache.archiva.repository.maven.content; import org.apache.archiva.common.filelock.FileLockManager; import org.apache.archiva.common.utils.FileUtils; +import org.apache.archiva.common.utils.VersionUtil; import org.apache.archiva.configuration.FileTypes; import org.apache.archiva.metadata.maven.MavenMetadataReader; import org.apache.archiva.metadata.repository.storage.RepositoryPathTranslator; @@ -186,30 +187,42 @@ public class ManagedDefaultRepositoryContent @Override public <T extends ContentItem> T adaptItem( Class<T> clazz, ContentItem item ) throws LayoutException { - if (clazz.isAssignableFrom( Version.class )) + try { - if ( !item.hasCharacteristic( Version.class ) ) + if ( clazz.isAssignableFrom( Version.class ) ) { - item.setCharacteristic( Version.class, createVersionFromPath( item.getAsset() ) ); + if ( !item.hasCharacteristic( Version.class ) ) + { + item.setCharacteristic( Version.class, createVersionFromPath( item.getAsset( ) ) ); + } + return (T) item.adapt( Version.class ); } - return (T) item.adapt( Version.class ); - } else if ( clazz.isAssignableFrom( Project.class )) { - if ( !item.hasCharacteristic( Project.class ) ) + else if ( clazz.isAssignableFrom( Project.class ) ) { - item.setCharacteristic( Project.class, createProjectFromPath( item.getAsset() ) ); + if ( !item.hasCharacteristic( Project.class ) ) + { + item.setCharacteristic( Project.class, createProjectFromPath( item.getAsset( ) ) ); + } + return (T) item.adapt( Project.class ); } - return (T) item.adapt( Project.class ); - } else if ( clazz.isAssignableFrom( Namespace.class )) { - if ( !item.hasCharacteristic( Namespace.class ) ) + else if ( clazz.isAssignableFrom( Namespace.class ) ) { - item.setCharacteristic( Namespace.class, createNamespaceFromPath( item.getAsset() ) ); + if ( !item.hasCharacteristic( Namespace.class ) ) + { + item.setCharacteristic( Namespace.class, createNamespaceFromPath( item.getAsset( ) ) ); + } + return (T) item.adapt( Namespace.class ); } - return (T) item.adapt( Namespace.class ); - } else if ( clazz.isAssignableFrom( Artifact.class )) { - if (!item.hasCharacteristic( Artifact.class )) { - item.setCharacteristic( Artifact.class, createArtifactFromPath( item.getAsset( ) ) ); + else if ( clazz.isAssignableFrom( Artifact.class ) ) + { + if ( !item.hasCharacteristic( Artifact.class ) ) + { + item.setCharacteristic( Artifact.class, createArtifactFromPath( item.getAsset( ) ) ); + } + return (T) item.adapt( Artifact.class ); } - return (T) item.adapt( Artifact.class ); + } catch (LayoutRuntimeException e) { + throw new LayoutException( e.getMessage( ), e ); } throw new LayoutException( "Could not convert item to class " + clazz); } @@ -593,6 +606,7 @@ public class ManagedDefaultRepositoryContent } } + private DataItem getDataItemFromPath( final StorageAsset artifactPath ) { final String contentType = getContentType( artifactPath ); @@ -644,13 +658,13 @@ public class ManagedDefaultRepositoryContent private ArtifactType artifactType = BaseArtifactTypes.MAIN; } - private ArtifactInfo getArtifactInfoFromPath( String genericVersion, StorageAsset path ) + private ArtifactInfo getArtifactInfoFromPath( final String genericVersion, final StorageAsset path ) { final ArtifactInfo info = new ArtifactInfo( ); info.asset = path; info.id = path.getParent( ).getParent( ).getName( ); final String fileName = path.getName( ); - if ( genericVersion.endsWith( "-" + SNAPSHOT ) ) + if ( VersionUtil.isGenericSnapshot( genericVersion ) ) { String baseVersion = StringUtils.substringBeforeLast( genericVersion, "-" + SNAPSHOT ); String prefix = info.id + "-" + baseVersion + "-"; @@ -722,7 +736,7 @@ public class ManagedDefaultRepositoryContent else { String prefix = info.id + "-" + genericVersion; - if ( fileName.startsWith( prefix ) ) + if ( fileName.startsWith( prefix + "-") ) { info.version = genericVersion; String classPostfix = StringUtils.removeStart( fileName, prefix ); @@ -737,6 +751,24 @@ public class ManagedDefaultRepositoryContent info.classifier = ""; info.remainder = classPostfix; } + } else if (fileName.startsWith(prefix + ".")) { + info.version = genericVersion; + info.remainder = StringUtils.removeStart( fileName, prefix ); + info.classifier = ""; + } else if (fileName.startsWith(info.id+"-")) { + String postFix = StringUtils.removeStart( fileName, info.id + "-" ); + String versionPart = StringUtils.substringBefore( postFix, "." ); + if (VersionUtil.isVersion(versionPart)) { + info.version = versionPart; + info.remainder = StringUtils.removeStart( postFix, versionPart ); + info.classifier = ""; + } else { + info.version = ""; + info.classifier = ""; + int dotPos = fileName.indexOf( "." ); + info.remainder = fileName.substring( dotPos ); + } + } else { @@ -747,10 +779,10 @@ public class ManagedDefaultRepositoryContent else { info.id = fileName; + info.version = ""; } log.debug( "Artifact does not match the version pattern {}", path ); info.artifactType = BaseArtifactTypes.UNKNOWN; - info.version = ""; info.classifier = ""; info.remainder = StringUtils.substringAfterLast( fileName, "." ); } @@ -1454,16 +1486,40 @@ public class ManagedDefaultRepositoryContent @Override public ContentItem toItem( String path ) throws LayoutException { + StorageAsset asset = getRepository( ).getAsset( path ); - if ( asset.isLeaf( ) ) - { - ItemSelector selector = getPathParser( ).toItemSelector( path ); - return getItem( selector ); - } - else - { - return getItemFromPath( asset ); + ContentItem item = getItemFromPath( asset ); + if (item instanceof DataItem) { + Artifact artifact = adaptItem( Artifact.class, item ); + if (asset.getParent()==null) { + throw new LayoutException( "Path too short for maven artifact "+path ); + } + String version = asset.getParent( ).getName( ); + if (asset.getParent().getParent()==null) { + throw new LayoutException( "Path too short for maven artifact " + path ); + } + String project = item.getAsset( ).getParent( ).getParent( ).getName( ); + DataItem dataItem = (DataItem) item; + if (StringUtils.isEmpty( dataItem.getExtension())) { + throw new LayoutException( "Missing type on maven artifact" ); + } + if (!project.equals(artifact.getId())) { + throw new LayoutException( "The maven artifact id "+artifact.getId() +" does not match the project id: " + project); + } + boolean versionIsGenericSnapshot = VersionUtil.isGenericSnapshot( version ); + boolean artifactVersionIsSnapshot = VersionUtil.isSnapshot( artifact.getArtifactVersion() ); + if ( versionIsGenericSnapshot && !artifactVersionIsSnapshot ) { + throw new LayoutException( "The maven artifact has no snapshot version in snapshot directory " + dataItem ); + } + if ( !versionIsGenericSnapshot && artifactVersionIsSnapshot) { + throw new LayoutException( "The maven artifact version " + artifact.getArtifactVersion() + " is a snapshot version but inside a non snapshot directory " + version ); + } + if ( !versionIsGenericSnapshot && !version.equals( artifact.getArtifactVersion() ) ) + { + throw new LayoutException( "The maven artifact version " + artifact.getArtifactVersion() + " does not match the version directory " + version ); + } } + return item; } @Override diff --git a/archiva-modules/archiva-maven/archiva-maven-repository/src/main/java/org/apache/archiva/repository/maven/content/MavenRepositoryRequestInfo.java b/archiva-modules/archiva-maven/archiva-maven-repository/src/main/java/org/apache/archiva/repository/maven/content/MavenRepositoryRequestInfo.java index 08afdcf..3a61455 100644 --- a/archiva-modules/archiva-maven/archiva-maven-repository/src/main/java/org/apache/archiva/repository/maven/content/MavenRepositoryRequestInfo.java +++ b/archiva-modules/archiva-maven/archiva-maven-repository/src/main/java/org/apache/archiva/repository/maven/content/MavenRepositoryRequestInfo.java @@ -20,6 +20,7 @@ package org.apache.archiva.repository.maven.content; import org.apache.archiva.model.ArtifactReference; import org.apache.archiva.repository.*; +import org.apache.archiva.repository.content.ItemSelector; import org.apache.archiva.repository.content.PathParser; import org.apache.archiva.repository.features.RepositoryFeature; import org.apache.archiva.repository.metadata.base.MetadataTools; @@ -82,6 +83,12 @@ public class MavenRepositoryRequestInfo implements RepositoryRequestInfo } } + @Override + public ItemSelector toItemSelector( String requestPath ) throws LayoutException + { + return repository.getContent( ).toItemSelector( requestPath ); + } + /** * <p> * Tests the path to see if it conforms to the expectations of a metadata request. @@ -275,12 +282,18 @@ public class MavenRepositoryRequestInfo implements RepositoryRequestInfo * Default layout is the only layout that can contain maven-metadata.xml files, and * if the managedRepository is layout legacy, this request would never occur. */ - return requestedPath; + if (requestedPath.startsWith( "/" )) { + return requestedPath; + } else + { + return "/"+requestedPath; + } } + + // Treat as an artifact reference. - ArtifactReference ref = toArtifactReference( referencedResource ); - String adjustedPath = repository.getContent().toPath( ref ); + String adjustedPath = repository.getContent( ).toPath( repository.getContent( ).toItem( requestedPath ) ); return adjustedPath + supportfile; } diff --git a/archiva-modules/archiva-maven/archiva-maven-repository/src/main/java/org/apache/archiva/repository/maven/metadata/storage/Maven2RepositoryStorage.java b/archiva-modules/archiva-maven/archiva-maven-repository/src/main/java/org/apache/archiva/repository/maven/metadata/storage/Maven2RepositoryStorage.java index 51dec60..e1c61aa 100644 --- a/archiva-modules/archiva-maven/archiva-maven-repository/src/main/java/org/apache/archiva/repository/maven/metadata/storage/Maven2RepositoryStorage.java +++ b/archiva-modules/archiva-maven/archiva-maven-repository/src/main/java/org/apache/archiva/repository/maven/metadata/storage/Maven2RepositoryStorage.java @@ -644,6 +644,98 @@ public class Maven2RepositoryStorage } } + @Override + public ItemSelector applyServerSideRelocation(ManagedRepository managedRepository, ItemSelector artifactSelector) + throws ProxyDownloadException { + if ("pom".equals(artifactSelector.getType())) { + return artifactSelector; + } + + // Build the artifact POM reference + BaseRepositoryContentLayout layout; + try + { + layout = managedRepository.getContent( ).getLayout( BaseRepositoryContentLayout.class ); + } + catch ( LayoutException e ) + { + throw new ProxyDownloadException( "Could not set layout " + e.getMessage( ), new HashMap<>( ) ); + } + + RepositoryType repositoryType = managedRepository.getType(); + if (!proxyRegistry.hasHandler(repositoryType)) { + throw new ProxyDownloadException("No proxy handler found for repository type " + repositoryType, new HashMap<>()); + } + + + + ItemSelector selector = ArchivaItemSelector.builder( ) + .withNamespace( artifactSelector.getNamespace( ) ) + .withProjectId( artifactSelector.getArtifactId( ) ) + .withArtifactId( artifactSelector.getArtifactId( ) ) + .withVersion( artifactSelector.getVersion( ) ) + .withArtifactVersion( artifactSelector.getVersion( ) ) + .withType( "pom" ).build( ); + + Artifact pom = layout.getArtifact( selector ); + + RepositoryProxyHandler proxyHandler = proxyRegistry.getHandler(repositoryType).get(0); + + // Get the artifact POM from proxied repositories if needed + proxyHandler.fetchFromProxies(managedRepository, pom); + + // Open and read the POM from the managed repo + + if (!pom.exists()) { + return artifactSelector; + } + + try { + // MavenXpp3Reader leaves the file open, so we need to close it ourselves. + + Model model; + try (Reader reader = Channels.newReader(pom.getAsset().getReadChannel(), Charset.defaultCharset().name())) { + model = MAVEN_XPP_3_READER.read(reader); + } + + DistributionManagement dist = model.getDistributionManagement(); + if (dist != null) { + Relocation relocation = dist.getRelocation(); + if (relocation != null) { + ArchivaItemSelector.Builder relocatedBuilder = ArchivaItemSelector.builder( ); + // artifact is relocated : update the repositoryPath + if (relocation.getGroupId() != null) { + relocatedBuilder.withNamespace( relocation.getGroupId( ) ); + } else { + relocatedBuilder.withNamespace( artifactSelector.getNamespace( ) ); + } + if (relocation.getArtifactId() != null) { + relocatedBuilder.withArtifactId( relocation.getArtifactId( ) ); + } else { + relocatedBuilder.withArtifactId( artifactSelector.getArtifactId( ) ); + } + if (relocation.getVersion() != null) + { + relocatedBuilder.withVersion( relocation.getVersion( ) ); + } else { + relocatedBuilder.withVersion( artifactSelector.getVersion( ) ); + } + return relocatedBuilder.withArtifactVersion( artifactSelector.getArtifactVersion( ) ) + .withClassifier( artifactSelector.getClassifier( ) ) + .withType( artifactSelector.getType( ) ) + .withProjectId( artifactSelector.getProjectId( ) ) + .withExtension( artifactSelector.getExtension( ) ) + .build( ); + } + } + } catch (IOException e) { + // Unable to read POM : ignore. + } catch (XmlPullParserException e) { + // Invalid POM : ignore + } + return artifactSelector; + } + @Override public String getFilePath(String requestPath, org.apache.archiva.repository.ManagedRepository managedRepository) { diff --git a/archiva-modules/archiva-maven/archiva-maven-repository/src/test/java/org/apache/archiva/repository/maven/content/ManagedDefaultRepositoryContentTest.java b/archiva-modules/archiva-maven/archiva-maven-repository/src/test/java/org/apache/archiva/repository/maven/content/ManagedDefaultRepositoryContentTest.java index 532997b..d86e759 100644 --- a/archiva-modules/archiva-maven/archiva-maven-repository/src/test/java/org/apache/archiva/repository/maven/content/ManagedDefaultRepositoryContentTest.java +++ b/archiva-modules/archiva-maven/archiva-maven-repository/src/test/java/org/apache/archiva/repository/maven/content/ManagedDefaultRepositoryContentTest.java @@ -1014,7 +1014,7 @@ public class ManagedDefaultRepositoryContentTest path = "/org/apache/maven/shared/maven-downloader/1.1/maven-downloader-1.1.jar"; item = repoContent.toItem( path ); assertNotNull( item ); - assertTrue( item instanceof Artifact ); + assertTrue( item instanceof DataItem ); } diff --git a/archiva-modules/archiva-maven/archiva-maven-repository/src/test/java/org/apache/archiva/repository/maven/content/MavenRepositoryRequestInfoTest.java b/archiva-modules/archiva-maven/archiva-maven-repository/src/test/java/org/apache/archiva/repository/maven/content/MavenRepositoryRequestInfoTest.java index 6e8fe08..7317e57 100644 --- a/archiva-modules/archiva-maven/archiva-maven-repository/src/test/java/org/apache/archiva/repository/maven/content/MavenRepositoryRequestInfoTest.java +++ b/archiva-modules/archiva-maven/archiva-maven-repository/src/test/java/org/apache/archiva/repository/maven/content/MavenRepositoryRequestInfoTest.java @@ -72,6 +72,9 @@ public class MavenRepositoryRequestInfoTest @Inject FileLockManager fileLockManager; + @Inject + MavenContentHelper mavenContentHelper; + private MavenRepositoryRequestInfo repoRequest; @@ -109,6 +112,8 @@ public class MavenRepositoryRequestInfoTest ManagedDefaultRepositoryContent repoContent = new ManagedDefaultRepositoryContent(repository, artifactMappingProviders, fileTypes, fileLockManager); //repoContent = (ManagedRepositoryContent) lookup( ManagedRepositoryContent.class, "default" ); repository.setContent(repoContent); + repoContent.setMavenContentHelper( mavenContentHelper ); + repoRequest = new MavenRepositoryRequestInfo(repository); } @@ -430,7 +435,7 @@ public class MavenRepositoryRequestInfoTest ManagedRepositoryContent repository = createManagedRepo( "default" ); // Test (artifact) default to default - dual extension - assertEquals( "org/project/example-presentation/3.2/example-presentation-3.2.xml.zip", + assertEquals( "/org/project/example-presentation/3.2/example-presentation-3.2.xml.zip", repoRequest.toNativePath( "org/project/example-presentation/3.2/example-presentation-3.2.xml.zip") ); } @@ -442,7 +447,7 @@ public class MavenRepositoryRequestInfoTest ManagedRepositoryContent repository = createManagedRepo( "default" ); // Test (metadata) default to default - assertEquals( "org/apache/derby/derby/10.2.2.0/maven-metadata.xml.sha1", + assertEquals( "/org/apache/derby/derby/10.2.2.0/maven-metadata.xml.sha1", repoRequest.toNativePath( "org/apache/derby/derby/10.2.2.0/maven-metadata.xml.sha1") ); } diff --git a/archiva-modules/archiva-web/archiva-security/src/test/java/org/apache/archiva/security/mock/MockBeanServices.java b/archiva-modules/archiva-web/archiva-security/src/test/java/org/apache/archiva/security/mock/MockBeanServices.java index 9fb1184..edb686b 100644 --- a/archiva-modules/archiva-web/archiva-security/src/test/java/org/apache/archiva/security/mock/MockBeanServices.java +++ b/archiva-modules/archiva-web/archiva-security/src/test/java/org/apache/archiva/security/mock/MockBeanServices.java @@ -31,6 +31,7 @@ import org.apache.archiva.components.taskqueue.TaskQueueException; import org.apache.archiva.repository.ManagedRepositoryContent; import org.apache.archiva.repository.ManagedRepository; import org.apache.archiva.metadata.audit.RepositoryListener; +import org.apache.archiva.repository.content.ItemSelector; import org.apache.archiva.scheduler.repository.model.RepositoryArchivaTaskScheduler; import org.apache.archiva.scheduler.repository.model.RepositoryTask; import org.apache.archiva.xml.XMLException; @@ -108,6 +109,12 @@ public class MockBeanServices } + @Override + public ItemSelector applyServerSideRelocation( ManagedRepository managedRepository, ItemSelector selector ) throws ProxyDownloadException + { + return null; + } + @Override public void deleteArtifact( MetadataRepository metadataRepository, String repositoryId, String namespace, diff --git a/archiva-modules/archiva-web/archiva-webdav/src/main/java/org/apache/archiva/webdav/ArchivaDavResourceFactory.java b/archiva-modules/archiva-web/archiva-webdav/src/main/java/org/apache/archiva/webdav/ArchivaDavResourceFactory.java index 8528b65..b83dcdb 100644 --- a/archiva-modules/archiva-web/archiva-webdav/src/main/java/org/apache/archiva/webdav/ArchivaDavResourceFactory.java +++ b/archiva-modules/archiva-web/archiva-webdav/src/main/java/org/apache/archiva/webdav/ArchivaDavResourceFactory.java @@ -68,6 +68,7 @@ import org.apache.archiva.repository.RepositoryRegistry; import org.apache.archiva.repository.RepositoryRequestInfo; import org.apache.archiva.repository.content.Artifact; import org.apache.archiva.repository.content.ContentItem; +import org.apache.archiva.repository.content.ItemSelector; import org.apache.archiva.repository.storage.fs.FilesystemStorage; import org.apache.archiva.repository.storage.StorageAsset; import org.apache.archiva.metadata.audit.AuditListener; @@ -791,22 +792,23 @@ public class ArchivaDavResourceFactory try { // Get the artifact reference in a layout neutral way. - ArtifactReference artifact = repositoryRequestInfo.toArtifactReference( path ); +// ArtifactReference artifact = repositoryRequestInfo.toArtifactReference( path ); + ItemSelector selector = repositoryRequestInfo.toItemSelector( path ); - if ( artifact != null ) + if ( selector != null ) { String repositoryLayout = managedRepository.getLayout(); RepositoryStorage repositoryStorage = this.applicationContext.getBean( "repositoryStorage#" + repositoryLayout, RepositoryStorage.class ); - repositoryStorage.applyServerSideRelocation( managedRepository, artifact ); + selector = repositoryStorage.applyServerSideRelocation( managedRepository, selector ); - StorageAsset proxiedFile = proxyHandler.fetchFromProxies( managedRepository, artifact ); + StorageAsset proxiedFile = proxyHandler.fetchFromProxies( managedRepository, selector ); - resource.setPath( managedRepository.getContent().toPath( artifact ) ); + resource.setPath( managedRepository.getContent().toPath( selector ) ); - log.debug( "Proxied artifact '{}:{}:{}'", artifact.getGroupId(), artifact.getArtifactId(), - artifact.getVersion() ); + log.debug( "Proxied artifact '{}:{}:{}:{}'", selector.getNamespace(), selector.getArtifactId(), + selector.getVersion(), selector.getArtifactVersion() ); return ( proxiedFile != null ); } diff --git a/archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/storage/RepositoryStorage.java b/archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/storage/RepositoryStorage.java index 6559be1..e563115 100644 --- a/archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/storage/RepositoryStorage.java +++ b/archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/storage/RepositoryStorage.java @@ -27,6 +27,7 @@ import org.apache.archiva.model.ArtifactReference; import org.apache.archiva.policies.ProxyDownloadException; import org.apache.archiva.repository.ManagedRepositoryContent; import org.apache.archiva.repository.ManagedRepository; +import org.apache.archiva.repository.content.ItemSelector; import org.apache.archiva.xml.XMLException; import java.io.IOException; @@ -75,6 +76,21 @@ public interface RepositoryStorage throws ProxyDownloadException; /** + * A relocation capable client will request the POM prior to the artifact, and will then read meta-data and do + * client side relocation. A simplier client (like maven 1) will only request the artifact and not use the + * metadatas. + * <p> + * For such clients, archiva does server-side relocation by reading itself the <relocation> element in + * metadatas and serving the expected artifact. + * @param managedRepository the used managed repository + * @param artifact the artifact reference + * @throws org.apache.archiva.policies.ProxyDownloadException + */ + ItemSelector applyServerSideRelocation( ManagedRepository managedRepository, ItemSelector selector ) + throws ProxyDownloadException; + + + /** * add an other method to evaluate real path as when receiving -SNAPSHOT (for maven storage) * request redirect to the last build * @param requestPath the web uri request diff --git a/archiva-modules/plugins/metadata-store-cassandra/src/test/java/org/apache/archiva/metadata/repository/cassandra/MockRepositoryStorage.java b/archiva-modules/plugins/metadata-store-cassandra/src/test/java/org/apache/archiva/metadata/repository/cassandra/MockRepositoryStorage.java index ab52023..d320ecb 100644 --- a/archiva-modules/plugins/metadata-store-cassandra/src/test/java/org/apache/archiva/metadata/repository/cassandra/MockRepositoryStorage.java +++ b/archiva-modules/plugins/metadata-store-cassandra/src/test/java/org/apache/archiva/metadata/repository/cassandra/MockRepositoryStorage.java @@ -37,6 +37,7 @@ import org.apache.archiva.policies.ProxyDownloadException; import org.apache.archiva.repository.ManagedRepositoryContent; import org.apache.archiva.repository.ManagedRepository; import org.apache.archiva.metadata.audit.RepositoryListener; +import org.apache.archiva.repository.content.ItemSelector; import org.apache.archiva.xml.XMLException; import java.io.IOException; @@ -113,6 +114,12 @@ public class MockRepositoryStorage } @Override + public ItemSelector applyServerSideRelocation( ManagedRepository managedRepository, ItemSelector selector ) throws ProxyDownloadException + { + return null; + } + + @Override public String getFilePath( String requestPath, org.apache.archiva.repository.ManagedRepository managedRepository ) { return null;
