This is an automated email from the ASF dual-hosted git repository.
cstamas pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/maven-resolver.git
The following commit(s) were added to refs/heads/master by this push:
new 26f00f3e [MRESOLVER-319] Parallel deploy, pt2 (#245)
26f00f3e is described below
commit 26f00f3edc9d391c42e32f7cf5e0b1a1326271f4
Author: Tamas Cservenak <[email protected]>
AuthorDate: Wed Feb 15 21:51:04 2023 +0100
[MRESOLVER-319] Parallel deploy, pt2 (#245)
In the mean time turns out we have two important "implementation details"
to obey, and behave correctly:
We need ONE transport to happen (go thru pipe fully) before we can pour the
rest onto transport
The HTTP transporter is equipped with "shared" caches for auth that is
heavily used by HttpClient to make decisions (ie. about re-attempting request).
Current code is quite complicated, but auth sharing happens when one transfer
task is done, and that step "inseminates" the cache to be used by others (and
skip the auth dance from that point onwards). Similar step happens by transfer
by issuing OPTIONS to remote to discover does it deals with WebDAV server or
not (is MKCOL dance needed [...]
We must ensure proper ordering, that was not ensured by origina
implementation
Clients when going for metadata are getting them as G -> A -> V (the
longest chain in case of snapshot maven plugin). Hence, we must ensure we
deploy the opposite order, and we start deploying next group ONLY when we are
fully done with previous group. Not ensuring this would leave a window of
opportunity of failed build, where for example A level md is present, but V
level not yet.
---
https://issues.apache.org/jira/browse/MRESOLVER-319
---
.../connector/basic/BasicRepositoryConnector.java | 119 ++++++++++++++++++---
1 file changed, 105 insertions(+), 14 deletions(-)
diff --git
a/maven-resolver-connector-basic/src/main/java/org/eclipse/aether/connector/basic/BasicRepositoryConnector.java
b/maven-resolver-connector-basic/src/main/java/org/eclipse/aether/connector/basic/BasicRepositoryConnector.java
index e7ff2e94..60e83b99 100644
---
a/maven-resolver-connector-basic/src/main/java/org/eclipse/aether/connector/basic/BasicRepositoryConnector.java
+++
b/maven-resolver-connector-basic/src/main/java/org/eclipse/aether/connector/basic/BasicRepositoryConnector.java
@@ -38,6 +38,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import org.eclipse.aether.ConfigurationProperties;
import org.eclipse.aether.RepositorySystemSession;
import org.eclipse.aether.RequestTrace;
+import org.eclipse.aether.metadata.Metadata;
import org.eclipse.aether.repository.RemoteRepository;
import org.eclipse.aether.spi.connector.ArtifactDownload;
import org.eclipse.aether.spi.connector.ArtifactUpload;
@@ -217,6 +218,8 @@ final class BasicRepositoryConnector
RunnableErrorForwarder errorForwarder = new RunnableErrorForwarder();
List<ChecksumAlgorithmFactory> checksumAlgorithmFactories =
layout.getChecksumAlgorithmFactories();
+ boolean first = true;
+
for ( MetadataDownload transfer : safeMetadataDownloads )
{
URI location = layout.getLocation( transfer.getMetadata(), false );
@@ -234,7 +237,15 @@ final class BasicRepositoryConnector
Runnable task = new GetTaskRunner( location, transfer.getFile(),
checksumPolicy,
checksumAlgorithmFactories, checksumLocations, null,
listener );
- executor.execute( errorForwarder.wrap( task ) );
+ if ( first )
+ {
+ task.run();
+ first = false;
+ }
+ else
+ {
+ executor.execute( errorForwarder.wrap( task ) );
+ }
}
for ( ArtifactDownload transfer : safeArtifactDownloads )
@@ -275,7 +286,15 @@ final class BasicRepositoryConnector
task = new GetTaskRunner( location, transfer.getFile(),
checksumPolicy,
checksumAlgorithmFactories, checksumLocations,
providedChecksums, listener );
}
- executor.execute( errorForwarder.wrap( task ) );
+ if ( first )
+ {
+ task.run();
+ first = false;
+ }
+ else
+ {
+ executor.execute( errorForwarder.wrap( task ) );
+ }
}
errorForwarder.await();
@@ -293,6 +312,8 @@ final class BasicRepositoryConnector
Executor executor = getExecutor( parallelPut ?
safeArtifactUploads.size() + safeMetadataUploads.size() : 1 );
RunnableErrorForwarder errorForwarder = new RunnableErrorForwarder();
+ boolean first = true;
+
for ( ArtifactUpload transfer : safeArtifactUploads )
{
URI location = layout.getLocation( transfer.getArtifact(), true );
@@ -306,29 +327,99 @@ final class BasicRepositoryConnector
Runnable task = new PutTaskRunner( location, transfer.getFile(),
transfer.getFileTransformer(),
checksumLocations, listener );
-
- executor.execute( errorForwarder.wrap( task ) );
+ if ( first )
+ {
+ task.run();
+ first = false;
+ }
+ else
+ {
+ executor.execute( errorForwarder.wrap( task ) );
+ }
}
errorForwarder.await(); // make sure all artifacts are PUT before we
go with Metadata
- for ( MetadataUpload transfer : safeMetadataUploads )
+ for ( List<? extends MetadataUpload> transferGroup : groupUploads(
safeMetadataUploads ) )
{
- URI location = layout.getLocation( transfer.getMetadata(), true );
+ for ( MetadataUpload transfer : transferGroup )
+ {
+ URI location = layout.getLocation( transfer.getMetadata(),
true );
- TransferResource resource = newTransferResource( location,
transfer.getFile(), transfer.getTrace() );
- TransferEvent.Builder builder = newEventBuilder( resource, true,
false );
- MetadataTransportListener listener = new
MetadataTransportListener( transfer, repository, builder );
+ TransferResource resource = newTransferResource( location,
transfer.getFile(), transfer.getTrace() );
+ TransferEvent.Builder builder = newEventBuilder( resource,
true, false );
+ MetadataTransportListener listener = new
MetadataTransportListener( transfer, repository, builder );
- List<RepositoryLayout.ChecksumLocation> checksumLocations =
- layout.getChecksumLocations( transfer.getMetadata(), true,
location );
+ List<RepositoryLayout.ChecksumLocation> checksumLocations =
+ layout.getChecksumLocations( transfer.getMetadata(),
true, location );
- Runnable task = new PutTaskRunner( location, transfer.getFile(),
checksumLocations, listener );
+ Runnable task = new PutTaskRunner( location,
transfer.getFile(), checksumLocations, listener );
+ if ( first )
+ {
+ task.run();
+ first = false;
+ }
+ else
+ {
+ executor.execute( errorForwarder.wrap( task ) );
+ }
+ }
- executor.execute( errorForwarder.wrap( task ) );
+ errorForwarder.await(); // make sure each group is done before
starting next group
}
+ }
- errorForwarder.await();
+ /**
+ * This method "groups" the Metadata to be uploaded by their level
(version, artifact, group and root). This is MUST
+ * as clients consume metadata in opposite order (root, group, artifact,
version), and hence, we must deploy and
+ * ensure (in case of parallel deploy) that all V level metadata is
deployed before we start deploying A level, etc.
+ */
+ private static List<List<MetadataUpload>> groupUploads( Collection<?
extends MetadataUpload> metadataUploads )
+ {
+ ArrayList<MetadataUpload> v = new ArrayList<>();
+ ArrayList<MetadataUpload> a = new ArrayList<>();
+ ArrayList<MetadataUpload> g = new ArrayList<>();
+ ArrayList<MetadataUpload> r = new ArrayList<>();
+
+ for ( MetadataUpload transfer : metadataUploads )
+ {
+ Metadata metadata = transfer.getMetadata();
+ if ( !"".equals( metadata.getVersion() ) )
+ {
+ v.add( transfer );
+ }
+ else if ( !"".equals( metadata.getArtifactId() ) )
+ {
+ a.add( transfer );
+ }
+ else if ( !"".equals( metadata.getGroupId() ) )
+ {
+ g.add( transfer );
+ }
+ else
+ {
+ r.add( transfer );
+ }
+ }
+
+ List<List<MetadataUpload>> result = new ArrayList<>( 4 );
+ if ( !v.isEmpty() )
+ {
+ result.add( v );
+ }
+ if ( !a.isEmpty() )
+ {
+ result.add( a );
+ }
+ if ( !g.isEmpty() )
+ {
+ result.add( g );
+ }
+ if ( !r.isEmpty() )
+ {
+ result.add( r );
+ }
+ return result;
}
private static <T> Collection<T> safe( Collection<T> items )
