Olivier Lamy
Wed, 20 Jan 2010 03:38:09 -0800
I just have a concurrency issue :
Exception in thread "pool-1-thread-3" java.util.ConcurrentModificationException
at
java.util.LinkedHashMap$LinkedHashIterator.nextEntry(LinkedHashMap.java:373)
at java.util.LinkedHashMap$EntryIterator.next(LinkedHashMap.java:392)
at java.util.LinkedHashMap$EntryIterator.next(LinkedHashMap.java:391)
at
org.apache.maven.cli.ConsoleMavenTransferListener.doProgress(ConsoleMavenTransferListener.java:57)
at
org.apache.maven.cli.AbstractMavenTransferListener.transferProgress(AbstractMavenTransferListener.java:106)
at
org.apache.maven.repository.legacy.TransferListenerAdapter.transferProgress(TransferListenerAdapter.java:128)
at
org.apache.maven.wagon.events.TransferEventSupport.fireTransferProgress(TransferEventSupport.java:124)
at
org.apache.maven.wagon.AbstractWagon.fireTransferProgress(AbstractWagon.java:505)
at org.apache.maven.wagon.AbstractWagon.transfer(AbstractWagon.java:490)
at
org.apache.maven.wagon.AbstractWagon.getTransfer(AbstractWagon.java:328)
at
org.apache.maven.wagon.AbstractWagon.getTransfer(AbstractWagon.java:299)
at
org.apache.maven.wagon.AbstractWagon.getTransfer(AbstractWagon.java:276)
at org.apache.maven.wagon.StreamWagon.getIfNewer(StreamWagon.java:97)
at org.apache.maven.wagon.StreamWagon.get(StreamWagon.java:61)
at
org.apache.maven.repository.legacy.DefaultWagonManager.getRemoteFile(DefaultWagonManager.java:309)
at
org.apache.maven.repository.legacy.DefaultWagonManager.getArtifact(DefaultWagonManager.java:98)
at
org.apache.maven.repository.legacy.DefaultWagonManager.getArtifact(DefaultWagonManager.java:129)
at
org.apache.maven.artifact.resolver.DefaultArtifactResolver.resolve(DefaultArtifactResolver.java:251)
at
org.apache.maven.artifact.resolver.DefaultArtifactResolver.access$000(DefaultArtifactResolver.java:73)
at
org.apache.maven.artifact.resolver.DefaultArtifactResolver$ResolveTask.run(DefaultArtifactResolver.java:686)
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:619)
But it's on a build server with a huge number of cpu ( solaris : 8 cores * 8 cpu per core). I'm testing a small change which looks to fix the issue. 2010/1/20 Olivier Lamy <ol...@apache.org>: > Nice and thanks !! > > -- > Olivier > > 2010/1/19 <bentm...@apache.org>: >> Author: bentmann >> Date: Tue Jan 19 22:25:12 2010 >> New Revision: 900982 >> >> URL: http://svn.apache.org/viewvc?rev=900982&view=rev >> Log: >> [MNG-4432] reimplement parallel artifacts download >> >> Modified: >> >> maven/maven-3/trunk/maven-compat/src/main/java/org/apache/maven/artifact/resolver/DefaultArtifactResolver.java >> >> maven/maven-3/trunk/maven-compat/src/main/java/org/apache/maven/repository/legacy/TransferListenerAdapter.java >> >> maven/maven-3/trunk/maven-embedder/src/main/java/org/apache/maven/cli/ConsoleMavenTransferListener.java >> >> Modified: >> maven/maven-3/trunk/maven-compat/src/main/java/org/apache/maven/artifact/resolver/DefaultArtifactResolver.java >> URL: >> http://svn.apache.org/viewvc/maven/maven-3/trunk/maven-compat/src/main/java/org/apache/maven/artifact/resolver/DefaultArtifactResolver.java?rev=900982&r1=900981&r2=900982&view=diff >> ============================================================================== >> --- >> maven/maven-3/trunk/maven-compat/src/main/java/org/apache/maven/artifact/resolver/DefaultArtifactResolver.java >> (original) >> +++ >> maven/maven-3/trunk/maven-compat/src/main/java/org/apache/maven/artifact/resolver/DefaultArtifactResolver.java >> Tue Jan 19 22:25:12 2010 >> @@ -24,6 +24,12 @@ >> import java.util.List; >> import java.util.Map; >> import java.util.Set; >> +import java.util.concurrent.CountDownLatch; >> +import java.util.concurrent.Executor; >> +import java.util.concurrent.ExecutorService; >> +import java.util.concurrent.LinkedBlockingQueue; >> +import java.util.concurrent.ThreadPoolExecutor; >> +import java.util.concurrent.TimeUnit; >> >> import org.apache.maven.artifact.Artifact; >> import org.apache.maven.artifact.factory.ArtifactFactory; >> @@ -97,6 +103,38 @@ >> @Requirement >> private LegacySupport legacySupport; >> >> + private final Executor executor; >> + >> + public DefaultArtifactResolver() >> + { >> + int threads = Integer.getInteger( "maven.artifact.threads", 5 >> ).intValue(); >> + if ( threads <= 1 ) >> + { >> + executor = new Executor() >> + { >> + public void execute( Runnable command ) >> + { >> + command.run(); >> + } >> + }; >> + } >> + else >> + { >> + executor = >> + new ThreadPoolExecutor( threads, threads, 3, >> TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>() ); >> + } >> + } >> + >> + �...@override >> + protected void finalize() >> + throws Throwable >> + { >> + if ( executor instanceof ExecutorService ) >> + { >> + ( (ExecutorService) executor ).shutdown(); >> + } >> + } >> + >> private void injectSession( RepositoryRequest request ) >> { >> MavenSession session = legacySupport.getSession(); >> @@ -558,41 +596,41 @@ >> { >> return result; >> } >> - >> + >> if ( result.getArtifactResolutionNodes() != null ) >> { >> - ArtifactResolutionRequest childRequest = new >> ArtifactResolutionRequest( request ); >> + ClassLoader classLoader = >> Thread.currentThread().getContextClassLoader(); >> + >> + CountDownLatch latch = new CountDownLatch( >> result.getArtifactResolutionNodes().size() ); >> >> for ( ResolutionNode node : result.getArtifactResolutionNodes() ) >> { >> Artifact artifact = node.getArtifact(); >> >> - try >> + if ( resolutionFilter == null || resolutionFilter.include( >> artifact ) ) >> { >> - if ( resolutionFilter == null || >> resolutionFilter.include( artifact ) ) >> - { >> - childRequest.setRemoteRepositories( >> node.getRemoteRepositories() ); >> + ArtifactResolutionRequest childRequest = new >> ArtifactResolutionRequest( request ); >> + childRequest.setRemoteRepositories( >> node.getRemoteRepositories() ); >> >> - resolve( artifact, childRequest, transferListener, >> false ); >> - } >> + executor.execute( new ResolveTask( classLoader, latch, >> artifact, transferListener, childRequest, >> + result ) ); >> } >> - catch ( ArtifactNotFoundException anfe ) >> - { >> - // These are cases where the artifact just isn't >> present in any of the remote repositories >> - // because it wasn't deployed, or it was deployed in >> the wrong place. >> - >> - result.addMissingArtifact( artifact ); >> - } >> - catch ( ArtifactResolutionException e ) >> + else >> { >> - // This is really a wagon TransferFailedException so >> something went wrong after we successfully >> - // retrieved the metadata. >> - >> - result.addErrorArtifactException( e ); >> + latch.countDown(); >> } >> } >> + try >> + { >> + latch.await(); >> + } >> + catch ( InterruptedException e ) >> + { >> + result.addErrorArtifactException( new >> ArtifactResolutionException( "Resolution interrupted", >> + >> rootArtifact, e ) ); >> + } >> } >> - >> + >> // We want to send the root artifact back in the result but we need >> to do this after the other dependencies >> // have been resolved. >> if ( request.isResolveRoot() ) >> @@ -612,4 +650,67 @@ >> { >> resolve( artifact, remoteRepositories, localRepository, null ); >> } >> + >> + private class ResolveTask >> + implements Runnable >> + { >> + >> + private final ClassLoader classLoader; >> + >> + private final CountDownLatch latch; >> + >> + private final Artifact artifact; >> + >> + private final TransferListener transferListener; >> + >> + private final ArtifactResolutionRequest request; >> + >> + private final ArtifactResolutionResult result; >> + >> + public ResolveTask( ClassLoader classLoader, CountDownLatch latch, >> Artifact artifact, TransferListener transferListener, >> + ArtifactResolutionRequest request, >> ArtifactResolutionResult result ) >> + { >> + this.classLoader = classLoader; >> + this.latch = latch; >> + this.artifact = artifact; >> + this.transferListener = transferListener; >> + this.request = request; >> + this.result = result; >> + } >> + >> + public void run() >> + { >> + try >> + { >> + Thread.currentThread().setContextClassLoader( classLoader ); >> + resolve( artifact, request, transferListener, false ); >> + } >> + catch ( ArtifactNotFoundException anfe ) >> + { >> + // These are cases where the artifact just isn't present in >> any of the remote repositories >> + // because it wasn't deployed, or it was deployed in the >> wrong place. >> + >> + synchronized ( result ) >> + { >> + result.addMissingArtifact( artifact ); >> + } >> + } >> + catch ( ArtifactResolutionException e ) >> + { >> + // This is really a wagon TransferFailedException so >> something went wrong after we successfully >> + // retrieved the metadata. >> + >> + synchronized ( result ) >> + { >> + result.addErrorArtifactException( e ); >> + } >> + } >> + finally >> + { >> + latch.countDown(); >> + } >> + } >> + >> + } >> + >> } >> >> Modified: >> maven/maven-3/trunk/maven-compat/src/main/java/org/apache/maven/repository/legacy/TransferListenerAdapter.java >> URL: >> http://svn.apache.org/viewvc/maven/maven-3/trunk/maven-compat/src/main/java/org/apache/maven/repository/legacy/TransferListenerAdapter.java?rev=900982&r1=900981&r2=900982&view=diff >> ============================================================================== >> --- >> maven/maven-3/trunk/maven-compat/src/main/java/org/apache/maven/repository/legacy/TransferListenerAdapter.java >> (original) >> +++ >> maven/maven-3/trunk/maven-compat/src/main/java/org/apache/maven/repository/legacy/TransferListenerAdapter.java >> Tue Jan 19 22:25:12 2010 >> @@ -34,11 +34,11 @@ >> implements TransferListener >> { >> >> - private ArtifactTransferListener listener; >> + private final ArtifactTransferListener listener; >> >> - private Map<Resource, ArtifactTransferResource> artifacts; >> + private final Map<Resource, ArtifactTransferResource> artifacts; >> >> - private Map<Resource, Long> transfers; >> + private final Map<Resource, Long> transfers; >> >> public static TransferListener newAdapter( ArtifactTransferListener >> listener ) >> { >> @@ -67,22 +67,34 @@ >> { >> ArtifactTransferEvent event = wrap( transferEvent ); >> >> - Long transferred = transfers.get( transferEvent.getResource() ); >> + Long transferred = null; >> + synchronized ( transfers ) >> + { >> + transferred = transfers.remove( transferEvent.getResource() ); >> + } >> if ( transferred != null ) >> { >> event.setTransferredBytes( transferred.longValue() ); >> } >> >> - listener.transferCompleted( event ); >> + synchronized ( artifacts ) >> + { >> + artifacts.remove( transferEvent.getResource() ); >> + } >> >> - artifacts.remove( transferEvent.getResource() ); >> - transfers.remove( transferEvent.getResource() ); >> + listener.transferCompleted( event ); >> } >> >> public void transferError( TransferEvent transferEvent ) >> { >> - artifacts.remove( transferEvent.getResource() ); >> - transfers.remove( transferEvent.getResource() ); >> + synchronized ( transfers ) >> + { >> + transfers.remove( transferEvent.getResource() ); >> + } >> + synchronized ( artifacts ) >> + { >> + artifacts.remove( transferEvent.getResource() ); >> + } >> } >> >> public void transferInitiated( TransferEvent transferEvent ) >> @@ -92,16 +104,20 @@ >> >> public void transferProgress( TransferEvent transferEvent, byte[] >> buffer, int length ) >> { >> - Long transferred = transfers.get( transferEvent.getResource() ); >> - if ( transferred == null ) >> - { >> - transferred = Long.valueOf( length ); >> - } >> - else >> + Long transferred; >> + synchronized ( transfers ) >> { >> - transferred = Long.valueOf( transferred.longValue() + length ); >> + transferred = transfers.get( transferEvent.getResource() ); >> + if ( transferred == null ) >> + { >> + transferred = Long.valueOf( length ); >> + } >> + else >> + { >> + transferred = Long.valueOf( transferred.longValue() + >> length ); >> + } >> + transfers.put( transferEvent.getResource(), transferred ); >> } >> - transfers.put( transferEvent.getResource(), transferred ); >> >> ArtifactTransferEvent event = wrap( transferEvent ); >> event.setDataBuffer( buffer ); >> @@ -153,15 +169,18 @@ >> } >> else >> { >> - ArtifactTransferResource artifact = artifacts.get( resource ); >> - >> - if ( artifact == null ) >> + synchronized ( artifacts ) >> { >> - artifact = new MavenArtifact( repository.getUrl(), resource >> ); >> - artifacts.put( resource, artifact ); >> - } >> + ArtifactTransferResource artifact = artifacts.get( resource >> ); >> >> - return artifact; >> + if ( artifact == null ) >> + { >> + artifact = new MavenArtifact( repository.getUrl(), >> resource ); >> + artifacts.put( resource, artifact ); >> + } >> + >> + return artifact; >> + } >> } >> } >> >> >> Modified: >> maven/maven-3/trunk/maven-embedder/src/main/java/org/apache/maven/cli/ConsoleMavenTransferListener.java >> URL: >> http://svn.apache.org/viewvc/maven/maven-3/trunk/maven-embedder/src/main/java/org/apache/maven/cli/ConsoleMavenTransferListener.java?rev=900982&r1=900981&r2=900982&view=diff >> ============================================================================== >> --- >> maven/maven-3/trunk/maven-embedder/src/main/java/org/apache/maven/cli/ConsoleMavenTransferListener.java >> (original) >> +++ >> maven/maven-3/trunk/maven-embedder/src/main/java/org/apache/maven/cli/ConsoleMavenTransferListener.java >> Tue Jan 19 22:25:12 2010 >> @@ -20,18 +20,27 @@ >> */ >> >> import java.io.PrintStream; >> +import java.util.Collections; >> +import java.util.LinkedHashMap; >> +import java.util.Map; >> >> import org.apache.maven.repository.ArtifactTransferEvent; >> +import org.apache.maven.repository.ArtifactTransferResource; >> >> /** >> * Console download progress meter. >> - * >> + * >> * @author <a href="mailto:br...@apache.org";>Brett Porter</a> >> */ >> class ConsoleMavenTransferListener >> extends AbstractMavenTransferListener >> { >> >> + private Map<ArtifactTransferResource, Long> downloads = >> + Collections.synchronizedMap( new >> LinkedHashMap<ArtifactTransferResource, Long>() ); >> + >> + private int lastLength; >> + >> public ConsoleMavenTransferListener( PrintStream out ) >> { >> super( out ); >> @@ -40,26 +49,69 @@ >> @Override >> protected void doProgress( ArtifactTransferEvent transferEvent ) >> { >> - long total = transferEvent.getResource().getContentLength(); >> - long complete = transferEvent.getTransferredBytes(); >> + ArtifactTransferResource resource = transferEvent.getResource(); >> + downloads.put( resource, Long.valueOf( >> transferEvent.getTransferredBytes() ) ); >> + >> + StringBuilder buffer = new StringBuilder( 64 ); >> >> - // TODO [BP]: Sys.out may no longer be appropriate, but will \r >> work with getLogger()? >> + for ( Map.Entry<ArtifactTransferResource, Long> entry : >> downloads.entrySet() ) >> + { >> + long total = entry.getKey().getContentLength(); >> + long complete = entry.getValue().longValue(); >> + >> + buffer.append( getStatus( complete, total ) ).append( " " ); >> + } >> + >> + int pad = lastLength - buffer.length(); >> + lastLength = buffer.length(); >> + pad( buffer, pad ); >> + buffer.append( '\r' ); >> + >> + out.print( buffer ); >> + } >> + >> + private String getStatus( long complete, long total ) >> + { >> if ( total >= 1024 ) >> { >> - out.print( toKB( complete ) + "/" + toKB( total ) + " KB " + >> "\r" ); >> + return toKB( complete ) + "/" + toKB( total ) + " KB "; >> } >> else if ( total >= 0 ) >> { >> - out.print( complete + "/" + total + " B " + "\r" ); >> + return complete + "/" + total + " B "; >> } >> else if ( complete >= 1024 ) >> { >> - out.print( toKB( complete ) + " KB " + "\r" ); >> + return toKB( complete ) + " KB "; >> } >> else >> { >> - out.print( complete + " B " + "\r" ); >> + return complete + " B "; >> } >> } >> >> + private void pad( StringBuilder buffer, int spaces ) >> + { >> + String block = " "; >> + while ( spaces > 0 ) >> + { >> + int n = Math.min( spaces, block.length() ); >> + buffer.append( block, 0, n ); >> + spaces -= n; >> + } >> + } >> + >> + �...@override >> + public void transferCompleted( ArtifactTransferEvent transferEvent ) >> + { >> + downloads.remove( transferEvent.getResource() ); >> + >> + StringBuilder buffer = new StringBuilder( 64 ); >> + pad( buffer, lastLength ); >> + buffer.append( '\r' ); >> + out.print( buffer ); >> + >> + super.transferCompleted( transferEvent ); >> + } >> + >> } >> >> >> > > > > -- > Olivier > -- Olivier --------------------------------------------------------------------- To unsubscribe, e-mail: dev-unsubscr...@maven.apache.org For additional commands, e-mail: dev-h...@maven.apache.org