Nice and thanks !! -- Olivier
2010/1/19 <bentm...@apache.org>: > Author: bentmann > Date: Tue Jan 19 22:25:12 2010 > New Revision: 900982 > > URL: http://svn.apache.org/viewvc?rev=900982&view=rev > Log: > [MNG-4432] reimplement parallel artifacts download > > Modified: > > maven/maven-3/trunk/maven-compat/src/main/java/org/apache/maven/artifact/resolver/DefaultArtifactResolver.java > > maven/maven-3/trunk/maven-compat/src/main/java/org/apache/maven/repository/legacy/TransferListenerAdapter.java > > maven/maven-3/trunk/maven-embedder/src/main/java/org/apache/maven/cli/ConsoleMavenTransferListener.java > > Modified: > maven/maven-3/trunk/maven-compat/src/main/java/org/apache/maven/artifact/resolver/DefaultArtifactResolver.java > URL: > http://svn.apache.org/viewvc/maven/maven-3/trunk/maven-compat/src/main/java/org/apache/maven/artifact/resolver/DefaultArtifactResolver.java?rev=900982&r1=900981&r2=900982&view=diff > ============================================================================== > --- > maven/maven-3/trunk/maven-compat/src/main/java/org/apache/maven/artifact/resolver/DefaultArtifactResolver.java > (original) > +++ > maven/maven-3/trunk/maven-compat/src/main/java/org/apache/maven/artifact/resolver/DefaultArtifactResolver.java > Tue Jan 19 22:25:12 2010 > @@ -24,6 +24,12 @@ > import java.util.List; > import java.util.Map; > import java.util.Set; > +import java.util.concurrent.CountDownLatch; > +import java.util.concurrent.Executor; > +import java.util.concurrent.ExecutorService; > +import java.util.concurrent.LinkedBlockingQueue; > +import java.util.concurrent.ThreadPoolExecutor; > +import java.util.concurrent.TimeUnit; > > import org.apache.maven.artifact.Artifact; > import org.apache.maven.artifact.factory.ArtifactFactory; > @@ -97,6 +103,38 @@ > @Requirement > private LegacySupport legacySupport; > > + private final Executor executor; > + > + public DefaultArtifactResolver() > + { > + int threads = Integer.getInteger( "maven.artifact.threads", 5 > ).intValue(); > + if ( threads <= 1 ) > + { > + executor = new Executor() > + { > + public void execute( Runnable command ) > + { > + command.run(); > + } > + }; > + } > + else > + { > + executor = > + new ThreadPoolExecutor( threads, threads, 3, > TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>() ); > + } > + } > + > + �...@override > + protected void finalize() > + throws Throwable > + { > + if ( executor instanceof ExecutorService ) > + { > + ( (ExecutorService) executor ).shutdown(); > + } > + } > + > private void injectSession( RepositoryRequest request ) > { > MavenSession session = legacySupport.getSession(); > @@ -558,41 +596,41 @@ > { > return result; > } > - > + > if ( result.getArtifactResolutionNodes() != null ) > { > - ArtifactResolutionRequest childRequest = new > ArtifactResolutionRequest( request ); > + ClassLoader classLoader = > Thread.currentThread().getContextClassLoader(); > + > + CountDownLatch latch = new CountDownLatch( > result.getArtifactResolutionNodes().size() ); > > for ( ResolutionNode node : result.getArtifactResolutionNodes() ) > { > Artifact artifact = node.getArtifact(); > > - try > + if ( resolutionFilter == null || resolutionFilter.include( > artifact ) ) > { > - if ( resolutionFilter == null || > resolutionFilter.include( artifact ) ) > - { > - childRequest.setRemoteRepositories( > node.getRemoteRepositories() ); > + ArtifactResolutionRequest childRequest = new > ArtifactResolutionRequest( request ); > + childRequest.setRemoteRepositories( > node.getRemoteRepositories() ); > > - resolve( artifact, childRequest, transferListener, > false ); > - } > + executor.execute( new ResolveTask( classLoader, latch, > artifact, transferListener, childRequest, > + result ) ); > } > - catch ( ArtifactNotFoundException anfe ) > - { > - // These are cases where the artifact just isn't present > in any of the remote repositories > - // because it wasn't deployed, or it was deployed in the > wrong place. > - > - result.addMissingArtifact( artifact ); > - } > - catch ( ArtifactResolutionException e ) > + else > { > - // This is really a wagon TransferFailedException so > something went wrong after we successfully > - // retrieved the metadata. > - > - result.addErrorArtifactException( e ); > + latch.countDown(); > } > } > + try > + { > + latch.await(); > + } > + catch ( InterruptedException e ) > + { > + result.addErrorArtifactException( new > ArtifactResolutionException( "Resolution interrupted", > + > rootArtifact, e ) ); > + } > } > - > + > // We want to send the root artifact back in the result but we need > to do this after the other dependencies > // have been resolved. > if ( request.isResolveRoot() ) > @@ -612,4 +650,67 @@ > { > resolve( artifact, remoteRepositories, localRepository, null ); > } > + > + private class ResolveTask > + implements Runnable > + { > + > + private final ClassLoader classLoader; > + > + private final CountDownLatch latch; > + > + private final Artifact artifact; > + > + private final TransferListener transferListener; > + > + private final ArtifactResolutionRequest request; > + > + private final ArtifactResolutionResult result; > + > + public ResolveTask( ClassLoader classLoader, CountDownLatch latch, > Artifact artifact, TransferListener transferListener, > + ArtifactResolutionRequest request, > ArtifactResolutionResult result ) > + { > + this.classLoader = classLoader; > + this.latch = latch; > + this.artifact = artifact; > + this.transferListener = transferListener; > + this.request = request; > + this.result = result; > + } > + > + public void run() > + { > + try > + { > + Thread.currentThread().setContextClassLoader( classLoader ); > + resolve( artifact, request, transferListener, false ); > + } > + catch ( ArtifactNotFoundException anfe ) > + { > + // These are cases where the artifact just isn't present in > any of the remote repositories > + // because it wasn't deployed, or it was deployed in the > wrong place. > + > + synchronized ( result ) > + { > + result.addMissingArtifact( artifact ); > + } > + } > + catch ( ArtifactResolutionException e ) > + { > + // This is really a wagon TransferFailedException so > something went wrong after we successfully > + // retrieved the metadata. > + > + synchronized ( result ) > + { > + result.addErrorArtifactException( e ); > + } > + } > + finally > + { > + latch.countDown(); > + } > + } > + > + } > + > } > > Modified: > maven/maven-3/trunk/maven-compat/src/main/java/org/apache/maven/repository/legacy/TransferListenerAdapter.java > URL: > http://svn.apache.org/viewvc/maven/maven-3/trunk/maven-compat/src/main/java/org/apache/maven/repository/legacy/TransferListenerAdapter.java?rev=900982&r1=900981&r2=900982&view=diff > ============================================================================== > --- > maven/maven-3/trunk/maven-compat/src/main/java/org/apache/maven/repository/legacy/TransferListenerAdapter.java > (original) > +++ > maven/maven-3/trunk/maven-compat/src/main/java/org/apache/maven/repository/legacy/TransferListenerAdapter.java > Tue Jan 19 22:25:12 2010 > @@ -34,11 +34,11 @@ > implements TransferListener > { > > - private ArtifactTransferListener listener; > + private final ArtifactTransferListener listener; > > - private Map<Resource, ArtifactTransferResource> artifacts; > + private final Map<Resource, ArtifactTransferResource> artifacts; > > - private Map<Resource, Long> transfers; > + private final Map<Resource, Long> transfers; > > public static TransferListener newAdapter( ArtifactTransferListener > listener ) > { > @@ -67,22 +67,34 @@ > { > ArtifactTransferEvent event = wrap( transferEvent ); > > - Long transferred = transfers.get( transferEvent.getResource() ); > + Long transferred = null; > + synchronized ( transfers ) > + { > + transferred = transfers.remove( transferEvent.getResource() ); > + } > if ( transferred != null ) > { > event.setTransferredBytes( transferred.longValue() ); > } > > - listener.transferCompleted( event ); > + synchronized ( artifacts ) > + { > + artifacts.remove( transferEvent.getResource() ); > + } > > - artifacts.remove( transferEvent.getResource() ); > - transfers.remove( transferEvent.getResource() ); > + listener.transferCompleted( event ); > } > > public void transferError( TransferEvent transferEvent ) > { > - artifacts.remove( transferEvent.getResource() ); > - transfers.remove( transferEvent.getResource() ); > + synchronized ( transfers ) > + { > + transfers.remove( transferEvent.getResource() ); > + } > + synchronized ( artifacts ) > + { > + artifacts.remove( transferEvent.getResource() ); > + } > } > > public void transferInitiated( TransferEvent transferEvent ) > @@ -92,16 +104,20 @@ > > public void transferProgress( TransferEvent transferEvent, byte[] buffer, > int length ) > { > - Long transferred = transfers.get( transferEvent.getResource() ); > - if ( transferred == null ) > - { > - transferred = Long.valueOf( length ); > - } > - else > + Long transferred; > + synchronized ( transfers ) > { > - transferred = Long.valueOf( transferred.longValue() + length ); > + transferred = transfers.get( transferEvent.getResource() ); > + if ( transferred == null ) > + { > + transferred = Long.valueOf( length ); > + } > + else > + { > + transferred = Long.valueOf( transferred.longValue() + length > ); > + } > + transfers.put( transferEvent.getResource(), transferred ); > } > - transfers.put( transferEvent.getResource(), transferred ); > > ArtifactTransferEvent event = wrap( transferEvent ); > event.setDataBuffer( buffer ); > @@ -153,15 +169,18 @@ > } > else > { > - ArtifactTransferResource artifact = artifacts.get( resource ); > - > - if ( artifact == null ) > + synchronized ( artifacts ) > { > - artifact = new MavenArtifact( repository.getUrl(), resource > ); > - artifacts.put( resource, artifact ); > - } > + ArtifactTransferResource artifact = artifacts.get( resource > ); > > - return artifact; > + if ( artifact == null ) > + { > + artifact = new MavenArtifact( repository.getUrl(), > resource ); > + artifacts.put( resource, artifact ); > + } > + > + return artifact; > + } > } > } > > > Modified: > maven/maven-3/trunk/maven-embedder/src/main/java/org/apache/maven/cli/ConsoleMavenTransferListener.java > URL: > http://svn.apache.org/viewvc/maven/maven-3/trunk/maven-embedder/src/main/java/org/apache/maven/cli/ConsoleMavenTransferListener.java?rev=900982&r1=900981&r2=900982&view=diff > ============================================================================== > --- > maven/maven-3/trunk/maven-embedder/src/main/java/org/apache/maven/cli/ConsoleMavenTransferListener.java > (original) > +++ > maven/maven-3/trunk/maven-embedder/src/main/java/org/apache/maven/cli/ConsoleMavenTransferListener.java > Tue Jan 19 22:25:12 2010 > @@ -20,18 +20,27 @@ > */ > > import java.io.PrintStream; > +import java.util.Collections; > +import java.util.LinkedHashMap; > +import java.util.Map; > > import org.apache.maven.repository.ArtifactTransferEvent; > +import org.apache.maven.repository.ArtifactTransferResource; > > /** > * Console download progress meter. > - * > + * > * @author <a href="mailto:br...@apache.org";>Brett Porter</a> > */ > class ConsoleMavenTransferListener > extends AbstractMavenTransferListener > { > > + private Map<ArtifactTransferResource, Long> downloads = > + Collections.synchronizedMap( new > LinkedHashMap<ArtifactTransferResource, Long>() ); > + > + private int lastLength; > + > public ConsoleMavenTransferListener( PrintStream out ) > { > super( out ); > @@ -40,26 +49,69 @@ > @Override > protected void doProgress( ArtifactTransferEvent transferEvent ) > { > - long total = transferEvent.getResource().getContentLength(); > - long complete = transferEvent.getTransferredBytes(); > + ArtifactTransferResource resource = transferEvent.getResource(); > + downloads.put( resource, Long.valueOf( > transferEvent.getTransferredBytes() ) ); > + > + StringBuilder buffer = new StringBuilder( 64 ); > > - // TODO [BP]: Sys.out may no longer be appropriate, but will \r work > with getLogger()? > + for ( Map.Entry<ArtifactTransferResource, Long> entry : > downloads.entrySet() ) > + { > + long total = entry.getKey().getContentLength(); > + long complete = entry.getValue().longValue(); > + > + buffer.append( getStatus( complete, total ) ).append( " " ); > + } > + > + int pad = lastLength - buffer.length(); > + lastLength = buffer.length(); > + pad( buffer, pad ); > + buffer.append( '\r' ); > + > + out.print( buffer ); > + } > + > + private String getStatus( long complete, long total ) > + { > if ( total >= 1024 ) > { > - out.print( toKB( complete ) + "/" + toKB( total ) + " KB " + > "\r" ); > + return toKB( complete ) + "/" + toKB( total ) + " KB "; > } > else if ( total >= 0 ) > { > - out.print( complete + "/" + total + " B " + "\r" ); > + return complete + "/" + total + " B "; > } > else if ( complete >= 1024 ) > { > - out.print( toKB( complete ) + " KB " + "\r" ); > + return toKB( complete ) + " KB "; > } > else > { > - out.print( complete + " B " + "\r" ); > + return complete + " B "; > } > } > > + private void pad( StringBuilder buffer, int spaces ) > + { > + String block = " "; > + while ( spaces > 0 ) > + { > + int n = Math.min( spaces, block.length() ); > + buffer.append( block, 0, n ); > + spaces -= n; > + } > + } > + > + �...@override > + public void transferCompleted( ArtifactTransferEvent transferEvent ) > + { > + downloads.remove( transferEvent.getResource() ); > + > + StringBuilder buffer = new StringBuilder( 64 ); > + pad( buffer, lastLength ); > + buffer.append( '\r' ); > + out.print( buffer ); > + > + super.transferCompleted( transferEvent ); > + } > + > } > > > -- Olivier --------------------------------------------------------------------- To unsubscribe, e-mail: dev-unsubscr...@maven.apache.org For additional commands, e-mail: dev-h...@maven.apache.org
