I just have a concurrency issue :
Exception in thread "pool-1-thread-3" java.util.ConcurrentModificationException
at
java.util.LinkedHashMap$LinkedHashIterator.nextEntry(LinkedHashMap.java:373)
at java.util.LinkedHashMap$EntryIterator.next(LinkedHashMap.java:392)
at java.util.LinkedHashMap$EntryIterator.next(LinkedHashMap.java:391)
at
org.apache.maven.cli.ConsoleMavenTransferListener.doProgress(ConsoleMavenTransferListener.java:57)
at
org.apache.maven.cli.AbstractMavenTransferListener.transferProgress(AbstractMavenTransferListener.java:106)
at
org.apache.maven.repository.legacy.TransferListenerAdapter.transferProgress(TransferListenerAdapter.java:128)
at
org.apache.maven.wagon.events.TransferEventSupport.fireTransferProgress(TransferEventSupport.java:124)
at
org.apache.maven.wagon.AbstractWagon.fireTransferProgress(AbstractWagon.java:505)
at org.apache.maven.wagon.AbstractWagon.transfer(AbstractWagon.java:490)
at
org.apache.maven.wagon.AbstractWagon.getTransfer(AbstractWagon.java:328)
at
org.apache.maven.wagon.AbstractWagon.getTransfer(AbstractWagon.java:299)
at
org.apache.maven.wagon.AbstractWagon.getTransfer(AbstractWagon.java:276)
at org.apache.maven.wagon.StreamWagon.getIfNewer(StreamWagon.java:97)
at org.apache.maven.wagon.StreamWagon.get(StreamWagon.java:61)
at
org.apache.maven.repository.legacy.DefaultWagonManager.getRemoteFile(DefaultWagonManager.java:309)
at
org.apache.maven.repository.legacy.DefaultWagonManager.getArtifact(DefaultWagonManager.java:98)
at
org.apache.maven.repository.legacy.DefaultWagonManager.getArtifact(DefaultWagonManager.java:129)
at
org.apache.maven.artifact.resolver.DefaultArtifactResolver.resolve(DefaultArtifactResolver.java:251)
at
org.apache.maven.artifact.resolver.DefaultArtifactResolver.access$000(DefaultArtifactResolver.java:73)
at
org.apache.maven.artifact.resolver.DefaultArtifactResolver$ResolveTask.run(DefaultArtifactResolver.java:686)
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:619)
But it's on a build server with a huge number of cpu ( solaris : 8
cores * 8 cpu per core).
I'm testing a small change which looks to fix the issue.
2010/1/20 Olivier Lamy <[email protected]>:
> Nice and thanks !!
>
> --
> Olivier
>
> 2010/1/19 <[email protected]>:
>> Author: bentmann
>> Date: Tue Jan 19 22:25:12 2010
>> New Revision: 900982
>>
>> URL: http://svn.apache.org/viewvc?rev=900982&view=rev
>> Log:
>> [MNG-4432] reimplement parallel artifacts download
>>
>> Modified:
>>
>> maven/maven-3/trunk/maven-compat/src/main/java/org/apache/maven/artifact/resolver/DefaultArtifactResolver.java
>>
>> maven/maven-3/trunk/maven-compat/src/main/java/org/apache/maven/repository/legacy/TransferListenerAdapter.java
>>
>> maven/maven-3/trunk/maven-embedder/src/main/java/org/apache/maven/cli/ConsoleMavenTransferListener.java
>>
>> Modified:
>> maven/maven-3/trunk/maven-compat/src/main/java/org/apache/maven/artifact/resolver/DefaultArtifactResolver.java
>> URL:
>> http://svn.apache.org/viewvc/maven/maven-3/trunk/maven-compat/src/main/java/org/apache/maven/artifact/resolver/DefaultArtifactResolver.java?rev=900982&r1=900981&r2=900982&view=diff
>> ==============================================================================
>> ---
>> maven/maven-3/trunk/maven-compat/src/main/java/org/apache/maven/artifact/resolver/DefaultArtifactResolver.java
>> (original)
>> +++
>> maven/maven-3/trunk/maven-compat/src/main/java/org/apache/maven/artifact/resolver/DefaultArtifactResolver.java
>> Tue Jan 19 22:25:12 2010
>> @@ -24,6 +24,12 @@
>> import java.util.List;
>> import java.util.Map;
>> import java.util.Set;
>> +import java.util.concurrent.CountDownLatch;
>> +import java.util.concurrent.Executor;
>> +import java.util.concurrent.ExecutorService;
>> +import java.util.concurrent.LinkedBlockingQueue;
>> +import java.util.concurrent.ThreadPoolExecutor;
>> +import java.util.concurrent.TimeUnit;
>>
>> import org.apache.maven.artifact.Artifact;
>> import org.apache.maven.artifact.factory.ArtifactFactory;
>> @@ -97,6 +103,38 @@
>> @Requirement
>> private LegacySupport legacySupport;
>>
>> + private final Executor executor;
>> +
>> + public DefaultArtifactResolver()
>> + {
>> + int threads = Integer.getInteger( "maven.artifact.threads", 5
>> ).intValue();
>> + if ( threads <= 1 )
>> + {
>> + executor = new Executor()
>> + {
>> + public void execute( Runnable command )
>> + {
>> + command.run();
>> + }
>> + };
>> + }
>> + else
>> + {
>> + executor =
>> + new ThreadPoolExecutor( threads, threads, 3,
>> TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>() );
>> + }
>> + }
>> +
>> + �...@override
>> + protected void finalize()
>> + throws Throwable
>> + {
>> + if ( executor instanceof ExecutorService )
>> + {
>> + ( (ExecutorService) executor ).shutdown();
>> + }
>> + }
>> +
>> private void injectSession( RepositoryRequest request )
>> {
>> MavenSession session = legacySupport.getSession();
>> @@ -558,41 +596,41 @@
>> {
>> return result;
>> }
>> -
>> +
>> if ( result.getArtifactResolutionNodes() != null )
>> {
>> - ArtifactResolutionRequest childRequest = new
>> ArtifactResolutionRequest( request );
>> + ClassLoader classLoader =
>> Thread.currentThread().getContextClassLoader();
>> +
>> + CountDownLatch latch = new CountDownLatch(
>> result.getArtifactResolutionNodes().size() );
>>
>> for ( ResolutionNode node : result.getArtifactResolutionNodes() )
>> {
>> Artifact artifact = node.getArtifact();
>>
>> - try
>> + if ( resolutionFilter == null || resolutionFilter.include(
>> artifact ) )
>> {
>> - if ( resolutionFilter == null ||
>> resolutionFilter.include( artifact ) )
>> - {
>> - childRequest.setRemoteRepositories(
>> node.getRemoteRepositories() );
>> + ArtifactResolutionRequest childRequest = new
>> ArtifactResolutionRequest( request );
>> + childRequest.setRemoteRepositories(
>> node.getRemoteRepositories() );
>>
>> - resolve( artifact, childRequest, transferListener,
>> false );
>> - }
>> + executor.execute( new ResolveTask( classLoader, latch,
>> artifact, transferListener, childRequest,
>> + result ) );
>> }
>> - catch ( ArtifactNotFoundException anfe )
>> - {
>> - // These are cases where the artifact just isn't
>> present in any of the remote repositories
>> - // because it wasn't deployed, or it was deployed in
>> the wrong place.
>> -
>> - result.addMissingArtifact( artifact );
>> - }
>> - catch ( ArtifactResolutionException e )
>> + else
>> {
>> - // This is really a wagon TransferFailedException so
>> something went wrong after we successfully
>> - // retrieved the metadata.
>> -
>> - result.addErrorArtifactException( e );
>> + latch.countDown();
>> }
>> }
>> + try
>> + {
>> + latch.await();
>> + }
>> + catch ( InterruptedException e )
>> + {
>> + result.addErrorArtifactException( new
>> ArtifactResolutionException( "Resolution interrupted",
>> +
>> rootArtifact, e ) );
>> + }
>> }
>> -
>> +
>> // We want to send the root artifact back in the result but we need
>> to do this after the other dependencies
>> // have been resolved.
>> if ( request.isResolveRoot() )
>> @@ -612,4 +650,67 @@
>> {
>> resolve( artifact, remoteRepositories, localRepository, null );
>> }
>> +
>> + private class ResolveTask
>> + implements Runnable
>> + {
>> +
>> + private final ClassLoader classLoader;
>> +
>> + private final CountDownLatch latch;
>> +
>> + private final Artifact artifact;
>> +
>> + private final TransferListener transferListener;
>> +
>> + private final ArtifactResolutionRequest request;
>> +
>> + private final ArtifactResolutionResult result;
>> +
>> + public ResolveTask( ClassLoader classLoader, CountDownLatch latch,
>> Artifact artifact, TransferListener transferListener,
>> + ArtifactResolutionRequest request,
>> ArtifactResolutionResult result )
>> + {
>> + this.classLoader = classLoader;
>> + this.latch = latch;
>> + this.artifact = artifact;
>> + this.transferListener = transferListener;
>> + this.request = request;
>> + this.result = result;
>> + }
>> +
>> + public void run()
>> + {
>> + try
>> + {
>> + Thread.currentThread().setContextClassLoader( classLoader );
>> + resolve( artifact, request, transferListener, false );
>> + }
>> + catch ( ArtifactNotFoundException anfe )
>> + {
>> + // These are cases where the artifact just isn't present in
>> any of the remote repositories
>> + // because it wasn't deployed, or it was deployed in the
>> wrong place.
>> +
>> + synchronized ( result )
>> + {
>> + result.addMissingArtifact( artifact );
>> + }
>> + }
>> + catch ( ArtifactResolutionException e )
>> + {
>> + // This is really a wagon TransferFailedException so
>> something went wrong after we successfully
>> + // retrieved the metadata.
>> +
>> + synchronized ( result )
>> + {
>> + result.addErrorArtifactException( e );
>> + }
>> + }
>> + finally
>> + {
>> + latch.countDown();
>> + }
>> + }
>> +
>> + }
>> +
>> }
>>
>> Modified:
>> maven/maven-3/trunk/maven-compat/src/main/java/org/apache/maven/repository/legacy/TransferListenerAdapter.java
>> URL:
>> http://svn.apache.org/viewvc/maven/maven-3/trunk/maven-compat/src/main/java/org/apache/maven/repository/legacy/TransferListenerAdapter.java?rev=900982&r1=900981&r2=900982&view=diff
>> ==============================================================================
>> ---
>> maven/maven-3/trunk/maven-compat/src/main/java/org/apache/maven/repository/legacy/TransferListenerAdapter.java
>> (original)
>> +++
>> maven/maven-3/trunk/maven-compat/src/main/java/org/apache/maven/repository/legacy/TransferListenerAdapter.java
>> Tue Jan 19 22:25:12 2010
>> @@ -34,11 +34,11 @@
>> implements TransferListener
>> {
>>
>> - private ArtifactTransferListener listener;
>> + private final ArtifactTransferListener listener;
>>
>> - private Map<Resource, ArtifactTransferResource> artifacts;
>> + private final Map<Resource, ArtifactTransferResource> artifacts;
>>
>> - private Map<Resource, Long> transfers;
>> + private final Map<Resource, Long> transfers;
>>
>> public static TransferListener newAdapter( ArtifactTransferListener
>> listener )
>> {
>> @@ -67,22 +67,34 @@
>> {
>> ArtifactTransferEvent event = wrap( transferEvent );
>>
>> - Long transferred = transfers.get( transferEvent.getResource() );
>> + Long transferred = null;
>> + synchronized ( transfers )
>> + {
>> + transferred = transfers.remove( transferEvent.getResource() );
>> + }
>> if ( transferred != null )
>> {
>> event.setTransferredBytes( transferred.longValue() );
>> }
>>
>> - listener.transferCompleted( event );
>> + synchronized ( artifacts )
>> + {
>> + artifacts.remove( transferEvent.getResource() );
>> + }
>>
>> - artifacts.remove( transferEvent.getResource() );
>> - transfers.remove( transferEvent.getResource() );
>> + listener.transferCompleted( event );
>> }
>>
>> public void transferError( TransferEvent transferEvent )
>> {
>> - artifacts.remove( transferEvent.getResource() );
>> - transfers.remove( transferEvent.getResource() );
>> + synchronized ( transfers )
>> + {
>> + transfers.remove( transferEvent.getResource() );
>> + }
>> + synchronized ( artifacts )
>> + {
>> + artifacts.remove( transferEvent.getResource() );
>> + }
>> }
>>
>> public void transferInitiated( TransferEvent transferEvent )
>> @@ -92,16 +104,20 @@
>>
>> public void transferProgress( TransferEvent transferEvent, byte[]
>> buffer, int length )
>> {
>> - Long transferred = transfers.get( transferEvent.getResource() );
>> - if ( transferred == null )
>> - {
>> - transferred = Long.valueOf( length );
>> - }
>> - else
>> + Long transferred;
>> + synchronized ( transfers )
>> {
>> - transferred = Long.valueOf( transferred.longValue() + length );
>> + transferred = transfers.get( transferEvent.getResource() );
>> + if ( transferred == null )
>> + {
>> + transferred = Long.valueOf( length );
>> + }
>> + else
>> + {
>> + transferred = Long.valueOf( transferred.longValue() +
>> length );
>> + }
>> + transfers.put( transferEvent.getResource(), transferred );
>> }
>> - transfers.put( transferEvent.getResource(), transferred );
>>
>> ArtifactTransferEvent event = wrap( transferEvent );
>> event.setDataBuffer( buffer );
>> @@ -153,15 +169,18 @@
>> }
>> else
>> {
>> - ArtifactTransferResource artifact = artifacts.get( resource );
>> -
>> - if ( artifact == null )
>> + synchronized ( artifacts )
>> {
>> - artifact = new MavenArtifact( repository.getUrl(), resource
>> );
>> - artifacts.put( resource, artifact );
>> - }
>> + ArtifactTransferResource artifact = artifacts.get( resource
>> );
>>
>> - return artifact;
>> + if ( artifact == null )
>> + {
>> + artifact = new MavenArtifact( repository.getUrl(),
>> resource );
>> + artifacts.put( resource, artifact );
>> + }
>> +
>> + return artifact;
>> + }
>> }
>> }
>>
>>
>> Modified:
>> maven/maven-3/trunk/maven-embedder/src/main/java/org/apache/maven/cli/ConsoleMavenTransferListener.java
>> URL:
>> http://svn.apache.org/viewvc/maven/maven-3/trunk/maven-embedder/src/main/java/org/apache/maven/cli/ConsoleMavenTransferListener.java?rev=900982&r1=900981&r2=900982&view=diff
>> ==============================================================================
>> ---
>> maven/maven-3/trunk/maven-embedder/src/main/java/org/apache/maven/cli/ConsoleMavenTransferListener.java
>> (original)
>> +++
>> maven/maven-3/trunk/maven-embedder/src/main/java/org/apache/maven/cli/ConsoleMavenTransferListener.java
>> Tue Jan 19 22:25:12 2010
>> @@ -20,18 +20,27 @@
>> */
>>
>> import java.io.PrintStream;
>> +import java.util.Collections;
>> +import java.util.LinkedHashMap;
>> +import java.util.Map;
>>
>> import org.apache.maven.repository.ArtifactTransferEvent;
>> +import org.apache.maven.repository.ArtifactTransferResource;
>>
>> /**
>> * Console download progress meter.
>> - *
>> + *
>> * @author <a href="mailto:[email protected]";>Brett Porter</a>
>> */
>> class ConsoleMavenTransferListener
>> extends AbstractMavenTransferListener
>> {
>>
>> + private Map<ArtifactTransferResource, Long> downloads =
>> + Collections.synchronizedMap( new
>> LinkedHashMap<ArtifactTransferResource, Long>() );
>> +
>> + private int lastLength;
>> +
>> public ConsoleMavenTransferListener( PrintStream out )
>> {
>> super( out );
>> @@ -40,26 +49,69 @@
>> @Override
>> protected void doProgress( ArtifactTransferEvent transferEvent )
>> {
>> - long total = transferEvent.getResource().getContentLength();
>> - long complete = transferEvent.getTransferredBytes();
>> + ArtifactTransferResource resource = transferEvent.getResource();
>> + downloads.put( resource, Long.valueOf(
>> transferEvent.getTransferredBytes() ) );
>> +
>> + StringBuilder buffer = new StringBuilder( 64 );
>>
>> - // TODO [BP]: Sys.out may no longer be appropriate, but will \r
>> work with getLogger()?
>> + for ( Map.Entry<ArtifactTransferResource, Long> entry :
>> downloads.entrySet() )
>> + {
>> + long total = entry.getKey().getContentLength();
>> + long complete = entry.getValue().longValue();
>> +
>> + buffer.append( getStatus( complete, total ) ).append( " " );
>> + }
>> +
>> + int pad = lastLength - buffer.length();
>> + lastLength = buffer.length();
>> + pad( buffer, pad );
>> + buffer.append( '\r' );
>> +
>> + out.print( buffer );
>> + }
>> +
>> + private String getStatus( long complete, long total )
>> + {
>> if ( total >= 1024 )
>> {
>> - out.print( toKB( complete ) + "/" + toKB( total ) + " KB " +
>> "\r" );
>> + return toKB( complete ) + "/" + toKB( total ) + " KB ";
>> }
>> else if ( total >= 0 )
>> {
>> - out.print( complete + "/" + total + " B " + "\r" );
>> + return complete + "/" + total + " B ";
>> }
>> else if ( complete >= 1024 )
>> {
>> - out.print( toKB( complete ) + " KB " + "\r" );
>> + return toKB( complete ) + " KB ";
>> }
>> else
>> {
>> - out.print( complete + " B " + "\r" );
>> + return complete + " B ";
>> }
>> }
>>
>> + private void pad( StringBuilder buffer, int spaces )
>> + {
>> + String block = " ";
>> + while ( spaces > 0 )
>> + {
>> + int n = Math.min( spaces, block.length() );
>> + buffer.append( block, 0, n );
>> + spaces -= n;
>> + }
>> + }
>> +
>> + �...@override
>> + public void transferCompleted( ArtifactTransferEvent transferEvent )
>> + {
>> + downloads.remove( transferEvent.getResource() );
>> +
>> + StringBuilder buffer = new StringBuilder( 64 );
>> + pad( buffer, lastLength );
>> + buffer.append( '\r' );
>> + out.print( buffer );
>> +
>> + super.transferCompleted( transferEvent );
>> + }
>> +
>> }
>>
>>
>>
>
>
>
> --
> Olivier
>
--
Olivier
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
