Repository: archiva Updated Branches: refs/heads/master 8462e8785 -> a4c023ab0
Implementing indexer methods Project: http://git-wip-us.apache.org/repos/asf/archiva/repo Commit: http://git-wip-us.apache.org/repos/asf/archiva/commit/a4c023ab Tree: http://git-wip-us.apache.org/repos/asf/archiva/tree/a4c023ab Diff: http://git-wip-us.apache.org/repos/asf/archiva/diff/a4c023ab Branch: refs/heads/master Commit: a4c023ab05d4739f07f7bd0b6a60538b7244dce3 Parents: 8462e87 Author: Martin Stockhammer <marti...@apache.org> Authored: Sun Nov 12 23:41:00 2017 +0100 Committer: Martin Stockhammer <marti...@apache.org> Committed: Sun Nov 12 23:41:00 2017 +0100 ---------------------------------------------------------------------- .../archiva-base/archiva-maven2-indexer/pom.xml | 10 + .../indexer/maven/MavenIndexManager.java | 664 +++++++++++++++++-- .../archiva/indexer/ArchivaIndexManager.java | 12 +- .../archiva/indexer/ArchivaIndexingContext.java | 2 +- .../indexer/IndexCreationFailedException.java | 50 ++ .../indexer/IndexUpdateFailedException.java | 50 ++ .../UnsupportedBaseContextException.java | 50 ++ 7 files changed, 775 insertions(+), 63 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/archiva/blob/a4c023ab/archiva-modules/archiva-base/archiva-maven2-indexer/pom.xml ---------------------------------------------------------------------- diff --git a/archiva-modules/archiva-base/archiva-maven2-indexer/pom.xml b/archiva-modules/archiva-base/archiva-maven2-indexer/pom.xml index bc141b6..5f999d7 100644 --- a/archiva-modules/archiva-base/archiva-maven2-indexer/pom.xml +++ b/archiva-modules/archiva-base/archiva-maven2-indexer/pom.xml @@ -43,6 +43,10 @@ <artifactId>archiva-repository-layer</artifactId> </dependency> <dependency> + <groupId>org.apache.archiva</groupId> + <artifactId>archiva-proxy-common</artifactId> + </dependency> + <dependency> <groupId>org.springframework</groupId> <artifactId>spring-context</artifactId> </dependency> @@ -97,6 +101,12 @@ </exclusions> </dependency> <dependency> + <groupId>org.apache.maven.wagon</groupId> + <artifactId>wagon-http</artifactId> + <scope>provided</scope> + </dependency> + + <dependency> <groupId>org.eclipse.sisu</groupId> <artifactId>org.eclipse.sisu.plexus</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/archiva/blob/a4c023ab/archiva-modules/archiva-base/archiva-maven2-indexer/src/main/java/org/apache/archiva/indexer/maven/MavenIndexManager.java ---------------------------------------------------------------------- diff --git a/archiva-modules/archiva-base/archiva-maven2-indexer/src/main/java/org/apache/archiva/indexer/maven/MavenIndexManager.java b/archiva-modules/archiva-base/archiva-maven2-indexer/src/main/java/org/apache/archiva/indexer/maven/MavenIndexManager.java index 12d8586..30cd3d8 100644 --- a/archiva-modules/archiva-base/archiva-maven2-indexer/src/main/java/org/apache/archiva/indexer/maven/MavenIndexManager.java +++ b/archiva-modules/archiva-base/archiva-maven2-indexer/src/main/java/org/apache/archiva/indexer/maven/MavenIndexManager.java @@ -19,153 +19,705 @@ package org.apache.archiva.indexer.maven; * under the License. */ +import org.apache.archiva.admin.model.RepositoryAdminException; +import org.apache.archiva.admin.model.beans.NetworkProxy; +import org.apache.archiva.admin.model.networkproxy.NetworkProxyAdmin; import org.apache.archiva.common.utils.PathUtil; import org.apache.archiva.configuration.ArchivaConfiguration; import org.apache.archiva.indexer.ArchivaIndexManager; import org.apache.archiva.indexer.ArchivaIndexingContext; +import org.apache.archiva.indexer.IndexCreationFailedException; +import org.apache.archiva.indexer.IndexUpdateFailedException; +import org.apache.archiva.indexer.UnsupportedBaseContextException; import org.apache.archiva.model.ArtifactReference; +import org.apache.archiva.proxy.common.WagonFactory; +import org.apache.archiva.proxy.common.WagonFactoryException; +import org.apache.archiva.proxy.common.WagonFactoryRequest; import org.apache.archiva.repository.ManagedRepository; +import org.apache.archiva.repository.PasswordCredentials; import org.apache.archiva.repository.RemoteRepository; import org.apache.archiva.repository.Repository; import org.apache.archiva.repository.RepositoryType; +import org.apache.archiva.repository.UnsupportedRepositoryTypeException; +import org.apache.archiva.repository.features.IndexCreationFeature; import org.apache.archiva.repository.features.RemoteIndexFeature; +import org.apache.commons.lang.StringUtils; +import org.apache.maven.index.DefaultScannerListener; import org.apache.maven.index.Indexer; +import org.apache.maven.index.IndexerEngine; +import org.apache.maven.index.Scanner; +import org.apache.maven.index.ScanningRequest; +import org.apache.maven.index.ScanningResult; import org.apache.maven.index.context.IndexCreator; import org.apache.maven.index.context.IndexingContext; +import org.apache.maven.index.packer.IndexPacker; +import org.apache.maven.index.packer.IndexPackingRequest; +import org.apache.maven.index.updater.IndexUpdateRequest; +import org.apache.maven.index.updater.IndexUpdater; +import org.apache.maven.index.updater.ResourceFetcher; import org.apache.maven.index_shaded.lucene.index.IndexFormatTooOldException; +import org.apache.maven.wagon.ConnectionException; +import org.apache.maven.wagon.ResourceDoesNotExistException; +import org.apache.maven.wagon.StreamWagon; +import org.apache.maven.wagon.TransferFailedException; +import org.apache.maven.wagon.Wagon; +import org.apache.maven.wagon.authentication.AuthenticationException; +import org.apache.maven.wagon.authentication.AuthenticationInfo; +import org.apache.maven.wagon.authorization.AuthorizationException; +import org.apache.maven.wagon.events.TransferEvent; +import org.apache.maven.wagon.events.TransferListener; +import org.apache.maven.wagon.proxy.ProxyInfo; +import org.apache.maven.wagon.shared.http.AbstractHttpClientWagon; +import org.apache.maven.wagon.shared.http.HttpConfiguration; +import org.apache.maven.wagon.shared.http.HttpMethodConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; import javax.inject.Inject; +import java.io.FileNotFoundException; import java.io.IOException; +import java.io.InputStream; +import java.net.MalformedURLException; import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentSkipListSet; /** * Maven implementation of index manager */ -@Service("archivaIndexManager#maven") -public class MavenIndexManager implements ArchivaIndexManager { +@Service( "archivaIndexManager#maven" ) +public class MavenIndexManager implements ArchivaIndexManager +{ - private static final Logger log = LoggerFactory.getLogger(MavenIndexManager.class); + private static final Logger log = LoggerFactory.getLogger( MavenIndexManager.class ); @Inject private Indexer indexer; @Inject + private IndexerEngine indexerEngine; + + @Inject private List<? extends IndexCreator> indexCreators; @Inject + private IndexPacker indexPacker; + + @Inject + private Scanner scanner; + + @Inject private ArchivaConfiguration archivaConfiguration; + @Inject + private WagonFactory wagonFactory; + + @Inject + private NetworkProxyAdmin networkProxyAdmin; + + @Inject + private IndexUpdater indexUpdater; + + private ConcurrentSkipListSet<Path> activeContexts = new ConcurrentSkipListSet<>( ); + + private static final int WAIT_TIME = 100; + private static final int MAX_WAIT = 10; + + + IndexingContext getMvnContext( ArchivaIndexingContext context ) throws UnsupportedBaseContextException + { + if ( !context.supports( IndexingContext.class ) ) + { + log.error( "The provided archiva index context does not support the maven IndexingContext" ); + throw new UnsupportedBaseContextException( "The context does not support the Maven IndexingContext" ); + } + return context.getBaseContext( IndexingContext.class ); + } + + private Path getIndexPath( ArchivaIndexingContext ctx ) + { + return PathUtil.getPathFromUri( ctx.getPath( ) ); + } + + @FunctionalInterface + interface IndexUpdateConsumer + { + + void accept( IndexingContext indexingContext ) throws IndexUpdateFailedException; + } + + private void executeUpdateFunction( ArchivaIndexingContext context, IndexUpdateConsumer function ) throws IndexUpdateFailedException + { + IndexingContext indexingContext = null; + try + { + indexingContext = getMvnContext( context ); + } + catch ( UnsupportedBaseContextException e ) + { + throw new IndexUpdateFailedException( "Maven index is not supported by this context", e ); + } + final Path ctxPath = getIndexPath( context ); + int loop = MAX_WAIT; + boolean active = false; + while ( loop-- > 0 && !active ) + { + active = activeContexts.add( ctxPath ); + try + { + Thread.currentThread( ).sleep( WAIT_TIME ); + } + catch ( InterruptedException e ) + { + // Ignore this + } + } + if ( active ) + { + try + { + function.accept( indexingContext ); + } + finally + { + activeContexts.remove( ctxPath ); + } + } + else + { + throw new IndexUpdateFailedException( "Timeout while waiting for index release on context " + context.getId( ) ); + } + } + @Override - public void pack(ArchivaIndexingContext context) { + public void pack( final ArchivaIndexingContext context ) throws IndexUpdateFailedException + { + executeUpdateFunction( context, indexingContext -> { + try + { + IndexPackingRequest request = new IndexPackingRequest( indexingContext, + indexingContext.acquireIndexSearcher( ).getIndexReader( ), + indexingContext.getIndexDirectoryFile( ) ); + indexPacker.packIndex( request ); + indexingContext.updateTimestamp( true ); + } + catch ( IOException e ) + { + log.error( "IOException while packing index of context " + context.getId( ) + ( StringUtils.isNotEmpty( e.getMessage( ) ) ? ": " + e.getMessage( ) : "" ) ); + throw new IndexUpdateFailedException( "IOException during update of " + context.getId( ), e ); + } + } + ); } @Override - public void scan(ArchivaIndexingContext context, boolean update) { + public void scan( final ArchivaIndexingContext context, final boolean update ) throws IndexUpdateFailedException + { + executeUpdateFunction( context, indexingContext -> { + DefaultScannerListener listener = new DefaultScannerListener( indexingContext, indexerEngine, update, null ); + ScanningRequest request = new ScanningRequest( indexingContext, listener ); + ScanningResult result = scanner.scan( request ); + if ( result.hasExceptions( ) ) + { + log.error( "Exceptions occured during index scan of " + context.getId( ) ); + result.getExceptions( ).stream( ).map( e -> e.getMessage( ) ).distinct( ).limit( 5 ).forEach( + s -> log.error( "Message: " + s ) + ); + } + } ); } @Override - public void update(ArchivaIndexingContext context, URI remoteUpdateUri, boolean fullUpdate) { + public void update( final ArchivaIndexingContext context, final URI remoteUpdateUri, final boolean fullUpdate ) throws IndexUpdateFailedException + { + log.info( "start download remote index for remote repository {}", context.getRepository( ).getId( ) ); + + if ( !( context.getRepository( ) instanceof RemoteRepository ) ) + { + throw new IndexUpdateFailedException( "The context is not associated to a remote repository " + context.getId( ) ); + } + final RemoteRepository remoteRepository = (RemoteRepository) context.getRepository( ); + + executeUpdateFunction( context, + indexingContext -> { + try + { + // create a temp directory to download files + Path tempIndexDirectory = Paths.get( indexingContext.getIndexDirectoryFile( ).getParent( ), ".tmpIndex" ); + Path indexCacheDirectory = Paths.get( indexingContext.getIndexDirectoryFile( ).getParent( ), ".indexCache" ); + Files.createDirectories( indexCacheDirectory ); + if ( Files.exists( tempIndexDirectory ) ) + { + org.apache.archiva.common.utils.FileUtils.deleteDirectory( tempIndexDirectory ); + } + Files.createDirectories( tempIndexDirectory ); + tempIndexDirectory.toFile( ).deleteOnExit( ); + String baseIndexUrl = indexingContext.getIndexUpdateUrl( ); + + String wagonProtocol = remoteUpdateUri.toURL( ).getProtocol( ); + + NetworkProxy networkProxy = null; + if ( remoteRepository.supportsFeature( RemoteIndexFeature.class ) ) + { + RemoteIndexFeature rif = remoteRepository.getFeature( RemoteIndexFeature.class ).get( ); + if ( StringUtils.isNotBlank( rif.getProxyId( ) ) ) + { + try + { + networkProxy = networkProxyAdmin.getNetworkProxy( rif.getProxyId( ) ); + } + catch ( RepositoryAdminException e ) + { + log.error( "Error occured while retrieving proxy {}", e.getMessage( ) ); + } + if ( networkProxy == null ) + { + log.warn( + "your remote repository is configured to download remote index trought a proxy we cannot find id:{}", + rif.getProxyId( ) ); + } + } + + final StreamWagon wagon = (StreamWagon) wagonFactory.getWagon( + new WagonFactoryRequest( wagonProtocol, remoteRepository.getExtraHeaders( ) ).networkProxy( + networkProxy ) + ); + int readTimeout = (int) rif.getDownloadTimeout( ).toMillis( ) * 1000; + wagon.setReadTimeout( readTimeout ); + wagon.setTimeout( (int) remoteRepository.getTimeout( ).toMillis( ) * 1000 ); + + if ( wagon instanceof AbstractHttpClientWagon ) + { + HttpConfiguration httpConfiguration = new HttpConfiguration( ); + HttpMethodConfiguration httpMethodConfiguration = new HttpMethodConfiguration( ); + httpMethodConfiguration.setUsePreemptive( true ); + httpMethodConfiguration.setReadTimeout( readTimeout ); + httpConfiguration.setGet( httpMethodConfiguration ); + AbstractHttpClientWagon.class.cast( wagon ).setHttpConfiguration( httpConfiguration ); + } + + wagon.addTransferListener( new DownloadListener( ) ); + ProxyInfo proxyInfo = null; + if ( networkProxy != null ) + { + proxyInfo = new ProxyInfo( ); + proxyInfo.setType( networkProxy.getProtocol( ) ); + proxyInfo.setHost( networkProxy.getHost( ) ); + proxyInfo.setPort( networkProxy.getPort( ) ); + proxyInfo.setUserName( networkProxy.getUsername( ) ); + proxyInfo.setPassword( networkProxy.getPassword( ) ); + } + AuthenticationInfo authenticationInfo = null; + if ( remoteRepository.getLoginCredentials( ) != null && ( remoteRepository.getLoginCredentials( ) instanceof PasswordCredentials ) ) + { + PasswordCredentials creds = (PasswordCredentials) remoteRepository.getLoginCredentials( ); + authenticationInfo = new AuthenticationInfo( ); + authenticationInfo.setUserName( creds.getUsername( ) ); + authenticationInfo.setPassword( new String( creds.getPassword( ) ) ); + } + wagon.connect( new org.apache.maven.wagon.repository.Repository( remoteRepository.getId( ), baseIndexUrl ), authenticationInfo, + proxyInfo ); + + Path indexDirectory = indexingContext.getIndexDirectoryFile( ).toPath( ); + if ( !Files.exists( indexDirectory ) ) + { + Files.createDirectories( indexDirectory ); + } + + ResourceFetcher resourceFetcher = + new WagonResourceFetcher( log, tempIndexDirectory, wagon, remoteRepository ); + IndexUpdateRequest request = new IndexUpdateRequest( indexingContext, resourceFetcher ); + request.setForceFullUpdate( fullUpdate ); + request.setLocalIndexCacheDir( indexCacheDirectory.toFile( ) ); + + indexUpdater.fetchAndUpdateIndex( request ); + + indexingContext.updateTimestamp( true ); + } + + } + catch ( AuthenticationException e ) + { + log.error( "Could not login to the remote proxy for updating index of {}", remoteRepository.getId( ), e ); + throw new IndexUpdateFailedException( "Login in to proxy failed while updating remote repository " + remoteRepository.getId( ), e ); + } + catch ( ConnectionException e ) + { + log.error( "Connection error during index update for remote repository {}", remoteRepository.getId( ), e ); + throw new IndexUpdateFailedException( "Connection error during index update for remote repository " + remoteRepository.getId( ), e ); + } + catch ( MalformedURLException e ) + { + log.error( "URL for remote index update of remote repository {} is not correct {}", remoteRepository.getId( ), remoteUpdateUri, e ); + throw new IndexUpdateFailedException( "URL for remote index update of repository is not correct " + remoteUpdateUri, e ); + } + catch ( IOException e ) + { + log.error( "IOException during index update of remote repository {}: {}", remoteRepository.getId( ), e.getMessage( ), e ); + throw new IndexUpdateFailedException( "IOException during index update of remote repository " + remoteRepository.getId( ) + + ( StringUtils.isNotEmpty( e.getMessage( ) ) ? ": " + e.getMessage( ) : "" ), e ); + } + catch ( WagonFactoryException e ) + { + log.error( "Wagon for remote index download of {} could not be created: {}", remoteRepository.getId( ), e.getMessage( ), e ); + throw new IndexUpdateFailedException( "Error while updating the remote index of " + remoteRepository.getId( ), e ); + } + } ); } @Override - public void addArtifactToIndex(ArchivaIndexingContext context, ArtifactReference artifactReference) { + public void addArtifactToIndex( ArchivaIndexingContext context, ArtifactReference artifactReference ) throws IndexUpdateFailedException + { } @Override - public void removeArtifactFromIndex(ArchivaIndexingContext context, ArtifactReference artifactReference) { + public void removeArtifactFromIndex( ArchivaIndexingContext context, ArtifactReference artifactReference ) throws IndexUpdateFailedException + { } @Override - public boolean supportsRepository(RepositoryType type) { - return false; + public boolean supportsRepository( RepositoryType type ) + { + return type == RepositoryType.MAVEN; } @Override - public ArchivaIndexingContext createContext(Repository remoteRepository) throws IOException { + public ArchivaIndexingContext createContext( Repository repository ) throws IndexCreationFailedException + { + + if ( repository.getType( ) != RepositoryType.MAVEN ) + { + throw new UnsupportedRepositoryTypeException( repository.getType( ) ); + } IndexingContext mvnCtx = null; - if (remoteRepository instanceof RemoteRepository) { - mvnCtx = createRemoteContext((RemoteRepository) remoteRepository); - } else if (remoteRepository instanceof ManagedRepository) { - // TODO: Implement managed repository index creation - mvnCtx = null; + try + { + if ( repository instanceof RemoteRepository ) + { + mvnCtx = createRemoteContext( (RemoteRepository) repository ); + } + else if ( repository instanceof ManagedRepository ) + { + mvnCtx = createManagedContext( (ManagedRepository) repository ); + } + } + catch ( IOException e ) + { + log.error( "IOException during context creation " + e.getMessage( ), e ); + throw new IndexCreationFailedException( "Could not create index context for repository " + repository.getId( ) + + ( StringUtils.isNotEmpty( e.getMessage( ) ) ? ": " + e.getMessage( ) : "" ), e ); } - MavenIndexContext context = new MavenIndexContext(remoteRepository, mvnCtx); - return null; + MavenIndexContext context = new MavenIndexContext( repository, mvnCtx ); + return context; } - private IndexingContext createRemoteContext(RemoteRepository remoteRepository) throws IOException { - Path appServerBase = archivaConfiguration.getAppServerBaseDir(); + private IndexingContext createRemoteContext( RemoteRepository remoteRepository ) throws IOException + { + Path appServerBase = archivaConfiguration.getAppServerBaseDir( ); - String contextKey = "remote-" + remoteRepository.getId(); + String contextKey = "remote-" + remoteRepository.getId( ); // create remote repository path - Path repoDir = appServerBase.resolve( "data").resolve( "remotes" ).resolve( remoteRepository.getId() ); - if ( !Files.exists(repoDir) ) + Path repoDir = appServerBase.resolve( "data" ).resolve( "remotes" ).resolve( remoteRepository.getId( ) ); + if ( !Files.exists( repoDir ) ) { - Files.createDirectories(repoDir); + Files.createDirectories( repoDir ); } - Path indexDirectory = null; + Path indexDirectory; // is there configured indexDirectory ? - if (remoteRepository.supportsFeature(RemoteIndexFeature.class)) { - RemoteIndexFeature rif = remoteRepository.getFeature(RemoteIndexFeature.class).get(); - indexDirectory = PathUtil.getPathFromUri(rif.getIndexUri()); - if (!indexDirectory.isAbsolute()) { - indexDirectory = repoDir.resolve(indexDirectory); + if ( remoteRepository.supportsFeature( RemoteIndexFeature.class ) ) + { + RemoteIndexFeature rif = remoteRepository.getFeature( RemoteIndexFeature.class ).get( ); + indexDirectory = PathUtil.getPathFromUri( rif.getIndexUri( ) ); + if ( !indexDirectory.isAbsolute( ) ) + { + indexDirectory = repoDir.resolve( indexDirectory ); } // if not configured use a default value - if (indexDirectory == null) { - indexDirectory = repoDir.resolve(".index"); + if ( indexDirectory == null ) + { + indexDirectory = repoDir.resolve( ".index" ); } - if (!Files.exists(indexDirectory)) { - Files.createDirectories(indexDirectory); + if ( !Files.exists( indexDirectory ) ) + { + Files.createDirectories( indexDirectory ); } - try { + String remoteIndexUrl = calculateIndexRemoteUrl( remoteRepository.getLocation( ), rif ); + try + { - return indexer.createIndexingContext(contextKey, remoteRepository.getId(), repoDir.toFile(), indexDirectory.toFile(), - remoteRepository.getLocation() == null ? null : remoteRepository.getLocation().toString(), - calculateIndexRemoteUrl(remoteRepository.getLocation(), rif), - true, false, - indexCreators); - } catch (IndexFormatTooOldException e) { + return getIndexingContext( remoteRepository, contextKey, repoDir, indexDirectory, remoteIndexUrl ); + } + catch ( IndexFormatTooOldException e ) + { // existing index with an old lucene format so we need to delete it!!! // delete it first then recreate it. - log.warn("the index of repository {} is too old we have to delete and recreate it", // - remoteRepository.getId()); - org.apache.archiva.common.utils.FileUtils.deleteDirectory(indexDirectory); - return indexer.createIndexingContext(contextKey, remoteRepository.getId(), repoDir.toFile(), indexDirectory.toFile(), - remoteRepository.getLocation() == null ? null : remoteRepository.getLocation().toString(), - calculateIndexRemoteUrl(remoteRepository.getLocation(), rif), - true, false, - indexCreators); + log.warn( "the index of repository {} is too old we have to delete and recreate it", // + remoteRepository.getId( ) ); + org.apache.archiva.common.utils.FileUtils.deleteDirectory( indexDirectory ); + return getIndexingContext( remoteRepository, contextKey, repoDir, indexDirectory, remoteIndexUrl ); + + } + } + else + { + throw new IOException( "No remote index defined" ); + } + } + + private IndexingContext getIndexingContext( Repository repository, String contextKey, Path repoDir, Path indexDirectory, String indexUrl ) throws IOException + { + return indexer.createIndexingContext( contextKey, repository.getId( ), repoDir.toFile( ), indexDirectory.toFile( ), + repository.getLocation( ) == null ? null : repository.getLocation( ).toString( ), + indexUrl, + true, false, + indexCreators ); + } + + private IndexingContext createManagedContext( ManagedRepository repository ) throws IOException + { + + IndexingContext context; + // take care first about repository location as can be relative + Path repositoryDirectory = PathUtil.getPathFromUri( repository.getLocation( ) ); + + if ( !repositoryDirectory.isAbsolute( ) ) + { + repositoryDirectory = + repositoryDirectory.resolve( "repositories" ).resolve( repositoryDirectory ); + } + + if ( !Files.exists( repositoryDirectory ) ) + { + try + { + Files.createDirectories( repositoryDirectory ); + } + catch ( IOException e ) + { + log.error( "Could not create directory {}", repositoryDirectory ); + } + } + + + if ( repository.supportsFeature( IndexCreationFeature.class ) ) + { + IndexCreationFeature icf = repository.getFeature( IndexCreationFeature.class ).get( ); + URI indexDir = icf.getIndexPath( ); + //File managedRepository = new File( repository.getLocation() ); + + Path indexDirectory = null; + if ( indexDir != null && !"".equals( indexDir.toString( ) ) ) + { + + indexDirectory = PathUtil.getPathFromUri( indexDir ); + // not absolute so create it in repository directory + if ( !indexDirectory.isAbsolute( ) ) + { + indexDirectory = repositoryDirectory.resolve( indexDirectory ); + } + icf.setIndexPath( indexDirectory.normalize( ).toUri( ) ); + } + else + { + indexDirectory = repositoryDirectory.resolve( ".indexer" ); + icf.setIndexPath( indexDirectory.toUri( ) ); + } + + if ( !Files.exists( indexDirectory ) ) + { + Files.createDirectories( indexDirectory ); + } + String indexUrl = repositoryDirectory.toUri( ).toURL( ).toExternalForm( ); + try + { + context = getIndexingContext( repository, repository.getId( ), repositoryDirectory, indexDirectory, indexUrl ); + context.setSearchable( repository.isScanned( ) ); } - } else { - throw new IOException("No remote index defined"); + catch ( IndexFormatTooOldException e ) + { + // existing index with an old lucene format so we need to delete it!!! + // delete it first then recreate it. + log.warn( "the index of repository {} is too old we have to delete and recreate it", // + repository.getId( ) ); + org.apache.archiva.common.utils.FileUtils.deleteDirectory( indexDirectory ); + context = getIndexingContext( repository, repository.getId( ), repositoryDirectory, indexDirectory, indexUrl ); + context.setSearchable( repository.isScanned( ) ); + } + return context; + } + else + { + throw new IOException( "No repository index defined" ); } } - private String calculateIndexRemoteUrl(URI baseUri, RemoteIndexFeature rif) { - if (rif.getIndexUri()==null) { - return baseUri.resolve(".index").toString(); - } else { - return baseUri.resolve(rif.getIndexUri()).toString(); + private String calculateIndexRemoteUrl( URI baseUri, RemoteIndexFeature rif ) + { + if ( rif.getIndexUri( ) == null ) + { + return baseUri.resolve( ".index" ).toString( ); + } + else + { + return baseUri.resolve( rif.getIndexUri( ) ).toString( ); + } + } + + private static final class DownloadListener + implements TransferListener + { + private Logger log = LoggerFactory.getLogger( getClass( ) ); + + private String resourceName; + + private long startTime; + + private int totalLength = 0; + + @Override + public void transferInitiated( TransferEvent transferEvent ) + { + startTime = System.currentTimeMillis( ); + resourceName = transferEvent.getResource( ).getName( ); + log.debug( "initiate transfer of {}", resourceName ); + } + + @Override + public void transferStarted( TransferEvent transferEvent ) + { + this.totalLength = 0; + resourceName = transferEvent.getResource( ).getName( ); + log.info( "start transfer of {}", transferEvent.getResource( ).getName( ) ); + } + + @Override + public void transferProgress( TransferEvent transferEvent, byte[] buffer, int length ) + { + log.debug( "transfer of {} : {}/{}", transferEvent.getResource( ).getName( ), buffer.length, length ); + this.totalLength += length; + } + + @Override + public void transferCompleted( TransferEvent transferEvent ) + { + resourceName = transferEvent.getResource( ).getName( ); + long endTime = System.currentTimeMillis( ); + log.info( "end of transfer file {} {} kb: {}s", transferEvent.getResource( ).getName( ), + this.totalLength / 1024, ( endTime - startTime ) / 1000 ); + } + + @Override + public void transferError( TransferEvent transferEvent ) + { + log.info( "error of transfer file {}: {}", transferEvent.getResource( ).getName( ), + transferEvent.getException( ).getMessage( ), transferEvent.getException( ) ); + } + + @Override + public void debug( String message ) + { + log.debug( "transfer debug {}", message ); } } + private static class WagonResourceFetcher + implements ResourceFetcher + { + + Logger log; + + Path tempIndexDirectory; + + Wagon wagon; + RemoteRepository remoteRepository; + + private WagonResourceFetcher( Logger log, Path tempIndexDirectory, Wagon wagon, + RemoteRepository remoteRepository ) + { + this.log = log; + this.tempIndexDirectory = tempIndexDirectory; + this.wagon = wagon; + this.remoteRepository = remoteRepository; + } + + @Override + public void connect( String id, String url ) + throws IOException + { + //no op + } + + @Override + public void disconnect( ) + throws IOException + { + // no op + } + + @Override + public InputStream retrieve( String name ) + throws IOException, FileNotFoundException + { + try + { + log.info( "index update retrieve file, name:{}", name ); + Path file = tempIndexDirectory.resolve( name ); + Files.deleteIfExists( file ); + file.toFile( ).deleteOnExit( ); + wagon.get( addParameters( name, remoteRepository ), file.toFile( ) ); + return Files.newInputStream( file ); + } + catch ( AuthorizationException | TransferFailedException e ) + { + throw new IOException( e.getMessage( ), e ); + } + catch ( ResourceDoesNotExistException e ) + { + FileNotFoundException fnfe = new FileNotFoundException( e.getMessage( ) ); + fnfe.initCause( e ); + throw fnfe; + } + } + + // FIXME remove crappy copy/paste + protected String addParameters( String path, RemoteRepository remoteRepository ) + { + if ( remoteRepository.getExtraParameters( ).isEmpty( ) ) + { + return path; + } + + boolean question = false; + + StringBuilder res = new StringBuilder( path == null ? "" : path ); + + for ( Map.Entry<String, String> entry : remoteRepository.getExtraParameters( ).entrySet( ) ) + { + if ( !question ) + { + res.append( '?' ).append( entry.getKey( ) ).append( '=' ).append( entry.getValue( ) ); + } + } + + return res.toString( ); + } + + } } http://git-wip-us.apache.org/repos/asf/archiva/blob/a4c023ab/archiva-modules/archiva-base/archiva-repository-api/src/main/java/org/apache/archiva/indexer/ArchivaIndexManager.java ---------------------------------------------------------------------- diff --git a/archiva-modules/archiva-base/archiva-repository-api/src/main/java/org/apache/archiva/indexer/ArchivaIndexManager.java b/archiva-modules/archiva-base/archiva-repository-api/src/main/java/org/apache/archiva/indexer/ArchivaIndexManager.java index 4663db6..b4bf63e 100644 --- a/archiva-modules/archiva-base/archiva-repository-api/src/main/java/org/apache/archiva/indexer/ArchivaIndexManager.java +++ b/archiva-modules/archiva-base/archiva-repository-api/src/main/java/org/apache/archiva/indexer/ArchivaIndexManager.java @@ -32,14 +32,14 @@ public interface ArchivaIndexManager { * Compresses the index to a more dense packed format. * @param context */ - void pack(ArchivaIndexingContext context) throws IOException; + void pack(ArchivaIndexingContext context) throws IndexUpdateFailedException; /** * Rescans the whole repository, this index is associated to. * @param context * @param update */ - void scan(ArchivaIndexingContext context, boolean update) throws IOException; + void scan(ArchivaIndexingContext context, boolean update) throws IndexUpdateFailedException; /** * Updates the index from the remote url. @@ -47,21 +47,21 @@ public interface ArchivaIndexManager { * @param remoteUpdateUri * @param fullUpdate */ - void update(ArchivaIndexingContext context, URI remoteUpdateUri, boolean fullUpdate) throws IOException; + void update(ArchivaIndexingContext context, URI remoteUpdateUri, boolean fullUpdate) throws IndexUpdateFailedException; /** * Adds a artifact to the index. * @param context * @param artifactReference */ - void addArtifactToIndex(ArchivaIndexingContext context, ArtifactReference artifactReference) throws IOException; + void addArtifactToIndex(ArchivaIndexingContext context, ArtifactReference artifactReference) throws IndexUpdateFailedException; /** * Removes a artifact from the index. * @param context * @param artifactReference */ - void removeArtifactFromIndex(ArchivaIndexingContext context, ArtifactReference artifactReference) throws IOException; + void removeArtifactFromIndex(ArchivaIndexingContext context, ArtifactReference artifactReference) throws IndexUpdateFailedException; /** @@ -76,5 +76,5 @@ public interface ArchivaIndexManager { * @param repository the repository for which the index context should be created * @return the index context */ - ArchivaIndexingContext createContext(Repository repository) throws IOException; + ArchivaIndexingContext createContext(Repository repository) throws IndexCreationFailedException; } http://git-wip-us.apache.org/repos/asf/archiva/blob/a4c023ab/archiva-modules/archiva-base/archiva-repository-api/src/main/java/org/apache/archiva/indexer/ArchivaIndexingContext.java ---------------------------------------------------------------------- diff --git a/archiva-modules/archiva-base/archiva-repository-api/src/main/java/org/apache/archiva/indexer/ArchivaIndexingContext.java b/archiva-modules/archiva-base/archiva-repository-api/src/main/java/org/apache/archiva/indexer/ArchivaIndexingContext.java index ee7cfc1..7f31b04 100644 --- a/archiva-modules/archiva-base/archiva-repository-api/src/main/java/org/apache/archiva/indexer/ArchivaIndexingContext.java +++ b/archiva-modules/archiva-base/archiva-repository-api/src/main/java/org/apache/archiva/indexer/ArchivaIndexingContext.java @@ -104,7 +104,7 @@ public interface ArchivaIndexingContext { * @return the instance of the given class representing this index * @throws UnsupportedOperationException if the implementation is not supported */ - <T> T getBaseContext(Class<T> clazz) throws UnsupportedOperationException; + <T> T getBaseContext(Class<T> clazz) throws UnsupportedBaseContextException; http://git-wip-us.apache.org/repos/asf/archiva/blob/a4c023ab/archiva-modules/archiva-base/archiva-repository-api/src/main/java/org/apache/archiva/indexer/IndexCreationFailedException.java ---------------------------------------------------------------------- diff --git a/archiva-modules/archiva-base/archiva-repository-api/src/main/java/org/apache/archiva/indexer/IndexCreationFailedException.java b/archiva-modules/archiva-base/archiva-repository-api/src/main/java/org/apache/archiva/indexer/IndexCreationFailedException.java new file mode 100644 index 0000000..27354e1 --- /dev/null +++ b/archiva-modules/archiva-base/archiva-repository-api/src/main/java/org/apache/archiva/indexer/IndexCreationFailedException.java @@ -0,0 +1,50 @@ +package org.apache.archiva.indexer; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * + */ +public class IndexCreationFailedException extends Exception +{ + public IndexCreationFailedException( ) + { + } + + public IndexCreationFailedException( String message ) + { + super( message ); + } + + public IndexCreationFailedException( String message, Throwable cause ) + { + super( message, cause ); + } + + public IndexCreationFailedException( Throwable cause ) + { + super( cause ); + } + + public IndexCreationFailedException( String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace ) + { + super( message, cause, enableSuppression, writableStackTrace ); + } +} http://git-wip-us.apache.org/repos/asf/archiva/blob/a4c023ab/archiva-modules/archiva-base/archiva-repository-api/src/main/java/org/apache/archiva/indexer/IndexUpdateFailedException.java ---------------------------------------------------------------------- diff --git a/archiva-modules/archiva-base/archiva-repository-api/src/main/java/org/apache/archiva/indexer/IndexUpdateFailedException.java b/archiva-modules/archiva-base/archiva-repository-api/src/main/java/org/apache/archiva/indexer/IndexUpdateFailedException.java new file mode 100644 index 0000000..bfa42f9 --- /dev/null +++ b/archiva-modules/archiva-base/archiva-repository-api/src/main/java/org/apache/archiva/indexer/IndexUpdateFailedException.java @@ -0,0 +1,50 @@ +package org.apache.archiva.indexer; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * + */ +public class IndexUpdateFailedException extends Exception +{ + public IndexUpdateFailedException( ) + { + } + + public IndexUpdateFailedException( String message ) + { + super( message ); + } + + public IndexUpdateFailedException( String message, Throwable cause ) + { + super( message, cause ); + } + + public IndexUpdateFailedException( Throwable cause ) + { + super( cause ); + } + + public IndexUpdateFailedException( String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace ) + { + super( message, cause, enableSuppression, writableStackTrace ); + } +} http://git-wip-us.apache.org/repos/asf/archiva/blob/a4c023ab/archiva-modules/archiva-base/archiva-repository-api/src/main/java/org/apache/archiva/indexer/UnsupportedBaseContextException.java ---------------------------------------------------------------------- diff --git a/archiva-modules/archiva-base/archiva-repository-api/src/main/java/org/apache/archiva/indexer/UnsupportedBaseContextException.java b/archiva-modules/archiva-base/archiva-repository-api/src/main/java/org/apache/archiva/indexer/UnsupportedBaseContextException.java new file mode 100644 index 0000000..92cfc20 --- /dev/null +++ b/archiva-modules/archiva-base/archiva-repository-api/src/main/java/org/apache/archiva/indexer/UnsupportedBaseContextException.java @@ -0,0 +1,50 @@ +package org.apache.archiva.indexer; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * + */ +public class UnsupportedBaseContextException extends Exception +{ + public UnsupportedBaseContextException( ) + { + } + + public UnsupportedBaseContextException( String message ) + { + super( message ); + } + + public UnsupportedBaseContextException( String message, Throwable cause ) + { + super( message, cause ); + } + + public UnsupportedBaseContextException( Throwable cause ) + { + super( cause ); + } + + public UnsupportedBaseContextException( String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace ) + { + super( message, cause, enableSuppression, writableStackTrace ); + } +}
