This is an automated email from the ASF dual-hosted git repository.
martin_s pushed a commit to branch feature/storage_refactoring
in repository https://gitbox.apache.org/repos/asf/archiva.git
The following commit(s) were added to refs/heads/feature/storage_refactoring by
this push:
new ecec848 Changing checksum handling in metadata
ecec848 is described below
commit ecec848cff05ca987e517c3fb8eef9a1e5f01121
Author: Martin Stockhammer <[email protected]>
AuthorDate: Sat Aug 24 17:09:57 2019 +0200
Changing checksum handling in metadata
---
archiva-modules/metadata/metadata-model/pom.xml | 6 +
.../archiva/metadata/model/ArtifactMetadata.java | 222 ++++++++++-----------
.../repository/AbstractMetadataRepositoryTest.java | 29 ++-
.../cassandra/CassandraArchivaManager.java | 2 +
.../cassandra/CassandraMetadataRepository.java | 182 +++++++++++++----
.../cassandra/DefaultCassandraArchivaManager.java | 50 +++++
.../cassandra/model/ArtifactMetadataModel.java | 20 ++
.../repository/cassandra/model/ColumnNames.java | 4 +-
.../cassandra/CassandraMetadataRepositoryTest.java | 4 +-
9 files changed, 356 insertions(+), 163 deletions(-)
diff --git a/archiva-modules/metadata/metadata-model/pom.xml
b/archiva-modules/metadata/metadata-model/pom.xml
index 4faf478..4d81ad9 100644
--- a/archiva-modules/metadata/metadata-model/pom.xml
+++ b/archiva-modules/metadata/metadata-model/pom.xml
@@ -33,9 +33,15 @@
<dependencies>
<dependency>
+ <groupId>org.apache.archiva</groupId>
+ <artifactId>archiva-checksum</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.apache.geronimo.specs</groupId>
<artifactId>geronimo-jpa_2.0_spec</artifactId>
</dependency>
</dependencies>
+
+
</project>
diff --git
a/archiva-modules/metadata/metadata-model/src/main/java/org/apache/archiva/metadata/model/ArtifactMetadata.java
b/archiva-modules/metadata/metadata-model/src/main/java/org/apache/archiva/metadata/model/ArtifactMetadata.java
index 8bb12d3..fd44feb 100644
---
a/archiva-modules/metadata/metadata-model/src/main/java/org/apache/archiva/metadata/model/ArtifactMetadata.java
+++
b/archiva-modules/metadata/metadata-model/src/main/java/org/apache/archiva/metadata/model/ArtifactMetadata.java
@@ -19,6 +19,7 @@ package org.apache.archiva.metadata.model;
* under the License.
*/
+import org.apache.archiva.checksum.ChecksumAlgorithm;
import sun.reflect.generics.repository.MethodRepository;
import javax.xml.bind.annotation.XmlRootElement;
@@ -28,6 +29,10 @@ import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.temporal.TemporalAccessor;
import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
/**
* Metadata stored in the content repository for a particular artifact.
Information that is shared between different
@@ -37,10 +42,9 @@ import java.util.Date;
* For more information, see the
* <a href="{@docRoot}/../metadata-content-model.html" target="_top">Metadata
Content Model</a>.
*/
-@XmlRootElement ( name = "artifactMetadata" )
+@XmlRootElement(name = "artifactMetadata")
public class ArtifactMetadata
- extends FacetedMetadata
-{
+ extends FacetedMetadata {
/**
@@ -57,21 +61,21 @@ public class ArtifactMetadata
/**
* The namespace of the project within the repository.
*
- * @see org.apache.archiva.metadata.model.ProjectMetadata#namespace
+ * @see ProjectMetadata#getNamespace()
*/
private String namespace;
/**
* The identifier of the project within the repository and namespace.
*
- * @see org.apache.archiva.metadata.model.ProjectMetadata#id
+ * @see ProjectMetadata#getId()
*/
private String project;
/**
* The version of the project. This may be more generalised than @{link
#version}.
*
- * @see org.apache.archiva.metadata.model.ProjectVersionMetadata#id
+ * @see ProjectVersionMetadata#getId()
*/
private String projectVersion;
@@ -93,193 +97,176 @@ public class ArtifactMetadata
private long size;
/**
- * The MD5 checksum of the artifact, if calculated.
+ * The list of checksums.
*/
- private String md5;
+ private Map<ChecksumAlgorithm, String> checksums = new HashMap<>();
- /**
- * The SHA-1 checksum of the artifact, if calculated.
- */
- private String sha1;
+ private String toStringValue = "";
+ private int lastHash = 0;
/**
* When the artifact was found in the repository storage and added to the
metadata content repository.
*/
private ZonedDateTime whenGathered;
- public String getId()
- {
+ public String getId() {
return id;
}
- public void setId( String id )
- {
+ public void setId(String id) {
this.id = id;
}
- public long getSize()
- {
+ public long getSize() {
return size;
}
- public void setSize( long size )
- {
+ public void setSize(long size) {
this.size = size;
}
- public String getVersion()
- {
+ public String getVersion() {
return version;
}
- public void setVersion( String version )
- {
+ public void setVersion(String version) {
this.version = version;
}
- public String getProjectVersion()
- {
+ public String getProjectVersion() {
return projectVersion;
}
- public void setProjectVersion( String projectVersion )
- {
+ public void setProjectVersion(String projectVersion) {
this.projectVersion = projectVersion;
}
- public void setFileLastModified( long fileLastModified )
- {
+ public void setFileLastModified(long fileLastModified) {
this.fileLastModified =
ZonedDateTime.ofInstant(Instant.ofEpochMilli(fileLastModified),
ModelInfo.STORAGE_TZ);
}
- public void setWhenGathered( ZonedDateTime whenGathered )
- {
+ public void setWhenGathered(ZonedDateTime whenGathered) {
this.whenGathered =
whenGathered.withZoneSameInstant(ModelInfo.STORAGE_TZ);
}
- public void setMd5( String md5 )
- {
- this.md5 = md5;
+ public void setMd5(String md5) {
+ this.checksums.put(ChecksumAlgorithm.MD5, md5);
}
- public void setSha1( String sha1 )
- {
- this.sha1 = sha1;
+ public void setSha1(String sha1) {
+ this.checksums.put(ChecksumAlgorithm.SHA1, sha1);
}
- public ZonedDateTime getWhenGathered()
- {
+ public ZonedDateTime getWhenGathered() {
return whenGathered;
}
- public String getMd5()
- {
- return md5;
+ public String getChecksum(ChecksumAlgorithm checksumAlgorithm) {
+ return checksums.get(checksumAlgorithm);
+ }
+
+ public void setChecksum(ChecksumAlgorithm algorithm, String checksumValue)
{
+ this.checksums.put(algorithm, checksumValue);
+ }
+
+ public Set<ChecksumAlgorithm> getChecksumTypes() {
+ return checksums.keySet();
}
- public String getSha1()
- {
- return sha1;
+ public Map<ChecksumAlgorithm,String> getChecksums() {
+ return this.checksums;
}
- public ZonedDateTime getFileLastModified()
- {
+ public void setChecksums(Map<ChecksumAlgorithm,String> checksums) {
+ this.checksums = checksums;
+ }
+
+ public String getMd5() {
+ return checksums.get(ChecksumAlgorithm.MD5);
+ }
+
+ public String getSha1() {
+ return checksums.get(ChecksumAlgorithm.SHA1);
+ }
+
+ public ZonedDateTime getFileLastModified() {
return fileLastModified;
}
- public String getNamespace()
- {
+ public String getNamespace() {
return namespace;
}
- public void setNamespace( String namespace )
- {
+ public void setNamespace(String namespace) {
this.namespace = namespace;
}
- public void setProject( String project )
- {
+ public void setProject(String project) {
this.project = project;
}
- public String getProject()
- {
+ public String getProject() {
return project;
}
- public String getRepositoryId()
- {
+ public String getRepositoryId() {
return repositoryId;
}
- public void setRepositoryId( String repositoryId )
- {
+ public void setRepositoryId(String repositoryId) {
this.repositoryId = repositoryId;
}
@Override
- public boolean equals( Object o )
- {
- if ( this == o )
- {
+ public boolean equals(Object o) {
+ if (this == o) {
return true;
}
- if ( o == null || getClass() != o.getClass() )
- {
+ if (o == null || getClass() != o.getClass()) {
return false;
}
ArtifactMetadata that = (ArtifactMetadata) o;
- if ( size != that.size )
- {
+ if (size != that.size) {
return false;
}
// Time equality by instant that means the point in time must match,
but not the time zone
- if ( fileLastModified != null
- ? !fileLastModified.toInstant().equals(
that.fileLastModified.toInstant() )
- : that.fileLastModified != null )
- {
+ if (fileLastModified != null
+ ?
!fileLastModified.toInstant().equals(that.fileLastModified.toInstant())
+ : that.fileLastModified != null) {
return false;
}
- if ( !id.equals( that.id ) )
- {
+ if (!id.equals(that.id)) {
return false;
}
- if ( md5 != null ? !md5.equals( that.md5 ) : that.md5 != null )
- {
- return false;
+ for ( Map.Entry<ChecksumAlgorithm, String> entry :
this.checksums.entrySet()) {
+ String thatChecksum = that.checksums.get(entry.getKey());
+ if (entry.getValue()!=null ?
!entry.getValue().equals(thatChecksum) : thatChecksum!=null) {
+ return false;
+ }
}
- if ( namespace != null ? !namespace.equals( that.namespace ) :
that.namespace != null )
- {
+ if (namespace != null ? !namespace.equals(that.namespace) :
that.namespace != null) {
return false;
}
- if ( project != null ? !project.equals( that.project ) : that.project
!= null )
- {
+ if (project != null ? !project.equals(that.project) : that.project !=
null) {
return false;
}
- if ( projectVersion != null ? !projectVersion.equals(
that.projectVersion ) : that.projectVersion != null )
- {
+ if (projectVersion != null ?
!projectVersion.equals(that.projectVersion) : that.projectVersion != null) {
return false;
}
/**
* We cannot compare in different repositories, if this is in here
- if ( !repositoryId.equals( that.repositoryId ) )
- {
- return false;
- }
+ if ( !repositoryId.equals( that.repositoryId ) )
+ {
+ return false;
+ }
**/
- if ( sha1 != null ? !sha1.equals( that.sha1 ) : that.sha1 != null )
- {
- return false;
- }
- if ( version != null ? !version.equals( that.version ) : that.version
!= null )
- {
+ if (version != null ? !version.equals(that.version) : that.version !=
null) {
return false;
}
- if ( whenGathered != null ? !whenGathered.toInstant().equals(
that.whenGathered.toInstant() ) : that.whenGathered != null )
- {
+ if (whenGathered != null ?
!whenGathered.toInstant().equals(that.whenGathered.toInstant()) :
that.whenGathered != null) {
return false;
}
@@ -287,28 +274,37 @@ public class ArtifactMetadata
}
@Override
- public int hashCode()
- {
+ public int hashCode() {
int result = id != null ? id.hashCode() : 0;
- result = 31 * result + ( repositoryId != null ?
repositoryId.hashCode() : 0 );
- result = 31 * result + ( namespace != null ? namespace.hashCode() : 0
);
- result = 31 * result + ( project != null ? project.hashCode() : 0 );
- result = 31 * result + ( projectVersion != null ?
projectVersion.hashCode() : 0 );
- result = 31 * result + ( version != null ? version.hashCode() : 0 );
- result = 31 * result + ( fileLastModified != null ?
fileLastModified.hashCode() : 0 );
- result = 31 * result + (int) ( size ^ ( size >>> 32 ) );
- result = 31 * result + ( md5 != null ? md5.hashCode() : 0 );
- result = 31 * result + ( sha1 != null ? sha1.hashCode() : 0 );
- result = 31 * result + ( whenGathered != null ?
whenGathered.hashCode() : 0 );
+ result = 31 * result + (repositoryId != null ? repositoryId.hashCode()
: 0);
+ result = 31 * result + (namespace != null ? namespace.hashCode() : 0);
+ result = 31 * result + (project != null ? project.hashCode() : 0);
+ result = 31 * result + (projectVersion != null ?
projectVersion.hashCode() : 0);
+ result = 31 * result + (version != null ? version.hashCode() : 0);
+ result = 31 * result + (fileLastModified != null ?
fileLastModified.hashCode() : 0);
+ result = 31 * result + (int) (size ^ (size >>> 32));
+ for (String checksum : checksums.values()) {
+ result = 31 * result + (checksum != null ? checksum.hashCode() :
0);
+ }
+ result = 31 * result + (whenGathered != null ? whenGathered.hashCode()
: 0);
return result;
}
+ /**
+ * Doing some hashing to avoid the expensive string concatenation.
+ */
@Override
- public String toString()
- {
- return "ArtifactMetadata{" + "id='" + id + '\'' + ", size=" + size +
", version='" + version + '\'' +
- ", fileLastModified=" + fileLastModified + ", whenGathered=" +
whenGathered + ", md5='" + md5 + '\'' +
- ", sha1='" + sha1 + '\'' + ", namespace='" + namespace + '\'' + ",
project='" + project + '\'' +
- ", projectVersion='" + projectVersion + '\'' + ", repositoryId='"
+ repositoryId + '\'' + '}';
+ public String toString() {
+ final int hashCode=hashCode();
+ if (hashCode!=lastHash) {
+ toStringValue = "ArtifactMetadata{" + "id='" + id + '\'' + ",
size=" + size + ", version='" + version + '\'' +
+ ", fileLastModified=" + fileLastModified + ",
whenGathered=" + whenGathered +
+ ", namespace='" + namespace + '\'' + ", project='" +
project + '\'' +
+ ", projectVersion='" + projectVersion + '\'' + ",
repositoryId='" + repositoryId + '\'' +
+ ", checksums=" + checksums.entrySet().stream().map(e ->
e.getKey() + ":" + e.getValue()).collect(Collectors.joining(",")) +
+ '}';
+ lastHash=hashCode;
+ }
+ return toStringValue;
}
}
diff --git
a/archiva-modules/metadata/metadata-repository-api/src/test/java/org/apache/archiva/metadata/repository/AbstractMetadataRepositoryTest.java
b/archiva-modules/metadata/metadata-repository-api/src/test/java/org/apache/archiva/metadata/repository/AbstractMetadataRepositoryTest.java
index a747b7f..c777097 100644
---
a/archiva-modules/metadata/metadata-repository-api/src/test/java/org/apache/archiva/metadata/repository/AbstractMetadataRepositoryTest.java
+++
b/archiva-modules/metadata/metadata-repository-api/src/test/java/org/apache/archiva/metadata/repository/AbstractMetadataRepositoryTest.java
@@ -80,6 +80,9 @@ public abstract class AbstractMetadataRepositoryTest
protected Logger log = LoggerFactory.getLogger( getClass( ) );
+ protected int assertMaxTries =10;
+ protected int assertRetrySleepMs=500;
+
/*
* Used by tryAssert to allow to throw exceptions in the lambda expression.
*/
@@ -91,7 +94,7 @@ public abstract class AbstractMetadataRepositoryTest
protected void tryAssert( AssertFunction func ) throws Exception
{
- tryAssert( func, 20, 500 );
+ tryAssert( func, assertMaxTries, assertRetrySleepMs );
}
@@ -960,15 +963,25 @@ public abstract class AbstractMetadataRepositoryTest
final ArtifactMetadata artifact1 = createArtifact( );
artifact1.setWhenGathered(ZonedDateTime.now().minusDays(1));
getRepository( ).updateArtifact( session, TEST_REPO_ID,
TEST_NAMESPACE, TEST_PROJECT, TEST_PROJECT_VERSION, artifact1 );
+
+ ZonedDateTime gatheredNow = ZonedDateTime.now();
final ArtifactMetadata artifact2 = createArtifact( );
- artifact2.setId(artifact2.getId()+"-2");
+ String artifact2Id = artifact2.getId() + "-2";
+ artifact2.setId(artifact2Id);
artifact2.setVersion(TEST_PROJECT_VERSION+"-2");
- artifact2.setWhenGathered(ZonedDateTime.now());
+ artifact2.setWhenGathered(gatheredNow);
getRepository( ).updateArtifact( session, TEST_REPO_ID,
TEST_NAMESPACE, TEST_PROJECT, TEST_PROJECT_VERSION, artifact2 );
final ArtifactMetadata artifact3 = createArtifact();
- artifact3.setId(artifact3.getId()+"-3");
+ String artifact3Id = artifact3.getId() + "-3";
+ artifact3.setId(artifact3Id);
artifact3.setVersion(TEST_PROJECT_VERSION+"-3");
- artifact3.setWhenGathered(ZonedDateTime.now().plusDays(1));
+ artifact3.setWhenGathered(gatheredNow.minusSeconds(5));
+
+ final ArtifactMetadata artifact4 = createArtifact();
+ artifact4.setId(artifact4.getId()+"-4");
+ artifact4.setVersion(TEST_PROJECT_VERSION+"-4");
+ artifact4.setWhenGathered(gatheredNow.plusDays(1));
+
getRepository( ).updateArtifact( session, TEST_REPO_ID,
TEST_NAMESPACE, TEST_PROJECT, TEST_PROJECT_VERSION, artifact3 );
session.save( );
@@ -980,10 +993,10 @@ public abstract class AbstractMetadataRepositoryTest
assertNotNull(stream);
List<ArtifactMetadata> artifacts =
stream.collect(Collectors.toList());
- assertEquals(1, artifacts.size());
-
+ assertEquals(2, artifacts.size());
+ assertEquals(artifact3Id, artifacts.get(0).getId());
+ assertEquals(artifact2Id, artifacts.get(1).getId());
- assertEquals( Collections.singletonList( artifact2 ),
artifacts );
} );
}
}
diff --git
a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraArchivaManager.java
b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraArchivaManager.java
index 45af483..41ac95a 100644
---
a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraArchivaManager.java
+++
b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraArchivaManager.java
@@ -56,4 +56,6 @@ public interface CassandraArchivaManager
String getDependencyFamilyName();
+ String getChecksumFamilyName();
+
}
diff --git
a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepository.java
b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepository.java
index 97a5753..768a94b 100644
---
a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepository.java
+++
b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepository.java
@@ -37,6 +37,7 @@ import me.prettyprint.hector.api.mutation.MutationResult;
import me.prettyprint.hector.api.mutation.Mutator;
import me.prettyprint.hector.api.query.QueryResult;
import me.prettyprint.hector.api.query.RangeSlicesQuery;
+import org.apache.archiva.checksum.ChecksumAlgorithm;
import org.apache.archiva.configuration.ArchivaConfiguration;
import org.apache.archiva.metadata.QueryParameter;
import org.apache.archiva.metadata.model.ArtifactMetadata;
@@ -76,9 +77,11 @@ import java.time.ZonedDateTime;
import java.util.*;
import java.util.function.BiFunction;
import java.util.function.Consumer;
+import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
+import static org.apache.archiva.metadata.model.ModelInfo.STORAGE_TZ;
import static
org.apache.archiva.metadata.repository.cassandra.CassandraUtils.*;
import static
org.apache.archiva.metadata.repository.cassandra.model.ColumnNames.*;
@@ -90,6 +93,7 @@ public class CassandraMetadataRepository
extends AbstractMetadataRepository implements MetadataRepository
{
+ private static final String ARTIFACT_METADATA_MODEL_KEY =
"artifactMetadataModel.key";
private Logger logger = LoggerFactory.getLogger( getClass() );
private ArchivaConfiguration configuration;
@@ -110,6 +114,8 @@ public class CassandraMetadataRepository
private final ColumnFamilyTemplate<String, String> dependencyTemplate;
+ private final ColumnFamilyTemplate<String, String> checksumTemplate;
+
private final Keyspace keyspace;
private final StringSerializer ss = StringSerializer.get();
@@ -163,6 +169,12 @@ public class CassandraMetadataRepository
//
StringSerializer.get(), //
StringSerializer.get() );
+
+ this.checksumTemplate = new ThriftColumnFamilyTemplate<>(
cassandraArchivaManager.getKeyspace(), //
+ cassandraArchivaManager.getChecksumFamilyName(),
+ //
+ StringSerializer.get(), //
+ StringSerializer.get() );
}
@@ -1036,6 +1048,77 @@ public class CassandraMetadataRepository
return projectVersionMetadata;
}
+ protected void recordChecksums( String repositoryId, String
artifactMetadataKey, Map<String, String> checksums)
+ {
+ if ( checksums == null || checksums.isEmpty() )
+ {
+ return;
+ }
+ Mutator<String> checksumMutator =
this.checksumTemplate.createMutator();
+ for ( Map.Entry<String, String> entry : checksums.entrySet())
+ {
+ // we don't care about the key as the real used one with the
projectVersionMetadata
+ String keyChecksums = UUID.randomUUID().toString();
+ String cfChecksums =
cassandraArchivaManager.getChecksumFamilyName();
+
+ addInsertion( checksumMutator, keyChecksums, cfChecksums,
ARTIFACT_METADATA_MODEL_KEY,
+ artifactMetadataKey );
+ addInsertion( checksumMutator, keyChecksums, cfChecksums,
CHECKSUM_ALG.toString(), entry.getKey());
+ addInsertion( checksumMutator, keyChecksums, cfChecksums,
CHECKSUM_VALUE.toString(),
+ entry.getValue() );
+ addInsertion(checksumMutator, keyChecksums, cfChecksums,
REPOSITORY_NAME.toString(), repositoryId);
+
+ }
+ checksumMutator.execute();
+ }
+
+ protected void removeChecksums( String artifactMetadataKey )
+ {
+
+ QueryResult<OrderedRows<String, String, String>> result =
+ HFactory.createRangeSlicesQuery(
cassandraArchivaManager.getKeyspace(), ss, ss, ss ) //
+ .setColumnFamily(
cassandraArchivaManager.getChecksumFamilyName() ) //
+ .setColumnNames( CHECKSUM_ALG.toString() ) //
+ .setRowCount( Integer.MAX_VALUE ) //
+ .addEqualsExpression(ARTIFACT_METADATA_MODEL_KEY,
artifactMetadataKey ) //
+ .execute();
+
+ if ( result.get().getCount() < 1 )
+ {
+ return;
+ }
+
+ for ( Row<String, String, String> row : result.get() )
+ {
+ this.checksumTemplate.deleteRow( row.getKey() );
+ }
+
+ }
+
+ protected Map<String, String> getChecksums( String artifactMetadataKey )
+ {
+ Map<String, String> checksums = new HashMap<>();
+
+ QueryResult<OrderedRows<String, String, String>> result =
+ HFactory.createRangeSlicesQuery(
cassandraArchivaManager.getKeyspace(), ss, ss, ss ) //
+ .setColumnFamily(
cassandraArchivaManager.getChecksumFamilyName() ) //
+ .setColumnNames( ARTIFACT_METADATA_MODEL_KEY,
REPOSITORY_NAME.toString(),
+ CHECKSUM_ALG.toString(),
CHECKSUM_VALUE.toString() ) //
+ .setRowCount( Integer.MAX_VALUE ) //
+ .addEqualsExpression(ARTIFACT_METADATA_MODEL_KEY,
artifactMetadataKey) //
+ .execute();
+ for ( Row<String, String, String> row : result.get() )
+ {
+ ColumnFamilyResult<String, String> columnFamilyResult =
+ this.checksumTemplate.queryColumns( row.getKey() );
+
+
checksums.put(columnFamilyResult.getString(CHECKSUM_ALG.toString()),
+ columnFamilyResult.getString(CHECKSUM_VALUE.toString()));
+ }
+
+ return checksums;
+ }
+
protected void recordMailingList( String projectVersionMetadataKey,
List<MailingList> mailingLists )
{
if ( mailingLists == null || mailingLists.isEmpty() )
@@ -1297,6 +1380,18 @@ public class CassandraMetadataRepository
return dependencies;
}
+ private Map<String, String> mapChecksums(Map<ChecksumAlgorithm,String>
checksums) {
+ return checksums.entrySet().stream().collect(Collectors.toMap(
+ e -> e.getKey().name(), e -> e.getValue()
+ ));
+ }
+
+ private Map<ChecksumAlgorithm, String>
mapChecksumsReverse(Map<String,String> checksums) {
+ return checksums.entrySet().stream().collect(Collectors.toMap(
+ e -> ChecksumAlgorithm.valueOf(e.getKey()), e -> e.getValue()
+ ));
+ }
+
@Override
public void updateArtifact( RepositorySession session, String
repositoryId, String namespaceId, String projectId, String projectVersion,
ArtifactMetadata artifactMeta )
@@ -1328,9 +1423,9 @@ public class CassandraMetadataRepository
updater.setLong( FILE_LAST_MODIFIED.toString(),
artifactMeta.getFileLastModified().toInstant().toEpochMilli());
updater.setLong( WHEN_GATHERED.toString(),
artifactMeta.getWhenGathered().toInstant().toEpochMilli() );
updater.setLong( SIZE.toString(), artifactMeta.getSize() );
- addUpdateStringValue( updater, MD5.toString(),
artifactMeta.getMd5() );
- addUpdateStringValue( updater, SHA1.toString(),
artifactMeta.getSha1() );
addUpdateStringValue( updater, VERSION.toString(),
artifactMeta.getVersion() );
+ removeChecksums(key);
+ recordChecksums(repositoryId, key,
mapChecksums(artifactMeta.getChecksums()));
this.artifactMetadataTemplate.update( updater );
}
else
@@ -1346,10 +1441,9 @@ public class CassandraMetadataRepository
.addInsertion( key, cf, column( VERSION.toString(),
artifactMeta.getVersion() ) ) //
.addInsertion( key, cf, column( FILE_LAST_MODIFIED.toString(),
artifactMeta.getFileLastModified().toInstant().toEpochMilli() ) ) //
.addInsertion( key, cf, column( SIZE.toString(),
artifactMeta.getSize() ) ) //
- .addInsertion( key, cf, column( MD5.toString(),
artifactMeta.getMd5() ) ) //
- .addInsertion( key, cf, column( SHA1.toString(),
artifactMeta.getSha1() ) ) //
.addInsertion( key, cf, column( WHEN_GATHERED.toString(),
artifactMeta.getWhenGathered().toInstant().toEpochMilli() ) )//
.execute();
+ recordChecksums(repositoryId, key,
mapChecksums(artifactMeta.getChecksums()));
}
key = new ProjectVersionMetadataModel.KeyBuilder() //
@@ -1397,6 +1491,7 @@ public class CassandraMetadataRepository
artifactMetadataModel.setFileLastModified(
artifactMeta.getFileLastModified() == null
?
ZonedDateTime.now().toInstant().toEpochMilli()
:
artifactMeta.getFileLastModified().toInstant().toEpochMilli() );
+
artifactMetadataModel.setChecksums(mapChecksums(artifactMeta.getChecksums()));
// now facets
updateFacets( artifactMeta, artifactMetadataModel );
@@ -1419,7 +1514,7 @@ public class CassandraMetadataRepository
.addEqualsExpression( PROJECT_VERSION.toString(), projectVersion )
//
.execute();
- final Set<String> versions = new HashSet<String>();
+ final Set<String> versions = new HashSet<>();
for ( Row<String, String, String> row : result.get() )
{
@@ -1806,15 +1901,16 @@ public class CassandraMetadataRepository
QueryResult<OrderedRows<String, String, Long>> result =
query.execute();
List<ArtifactMetadata> artifactMetadatas = new ArrayList<>(
result.get().getCount() );
-
- for ( Row<String, String, Long> row : result.get() )
- {
- ColumnSlice<String, Long> columnSlice = row.getColumnSlice();
- String repositoryName = getAsStringValue( columnSlice,
REPOSITORY_NAME.toString() );
- if ( StringUtils.equals( repositoryName, repositoryId ) )
- {
-
- artifactMetadatas.add( mapArtifactMetadataLongColumnSlice(
columnSlice ) );
+ Iterator<Row<String, String, Long>> keyIter = result.get().iterator();
+ if (keyIter.hasNext()) {
+ String key = keyIter.next().getKey();
+ for (Row<String, String, Long> row : result.get()) {
+ ColumnSlice<String, Long> columnSlice = row.getColumnSlice();
+ String repositoryName = getAsStringValue(columnSlice,
REPOSITORY_NAME.toString());
+ if (StringUtils.equals(repositoryName, repositoryId)) {
+
+
artifactMetadatas.add(mapArtifactMetadataLongColumnSlice(key, columnSlice));
+ }
}
}
@@ -1843,7 +1939,7 @@ public class CassandraMetadataRepository
}
- protected ArtifactMetadata mapArtifactMetadataLongColumnSlice(
ColumnSlice<String, Long> columnSlice )
+ protected ArtifactMetadata mapArtifactMetadataLongColumnSlice( String key,
ColumnSlice<String, Long> columnSlice )
{
ArtifactMetadata artifactMetadata = new ArtifactMetadata();
artifactMetadata.setNamespace( getAsStringValue( columnSlice,
NAMESPACE_ID.toString() ) );
@@ -1859,12 +1955,13 @@ public class CassandraMetadataRepository
Long whenGathered = getLongValue( columnSlice,
WHEN_GATHERED.toString() );
if ( whenGathered != null )
{
-
artifactMetadata.setWhenGathered(ZonedDateTime.ofInstant(Instant.ofEpochMilli(whenGathered),
ZoneId.of("GMT")));
+
artifactMetadata.setWhenGathered(ZonedDateTime.ofInstant(Instant.ofEpochMilli(whenGathered),
STORAGE_TZ));
}
+ artifactMetadata.setChecksums(mapChecksumsReverse(getChecksums(key)));
return artifactMetadata;
}
- protected ArtifactMetadata mapArtifactMetadataStringColumnSlice(
ColumnSlice<String, String> columnSlice )
+ protected ArtifactMetadata mapArtifactMetadataStringColumnSlice( String
key, ColumnSlice<String, String> columnSlice )
{
ArtifactMetadata artifactMetadata = new ArtifactMetadata();
artifactMetadata.setNamespace( getStringValue( columnSlice,
NAMESPACE_ID.toString() ) );
@@ -1880,8 +1977,9 @@ public class CassandraMetadataRepository
Long whenGathered = getAsLongValue( columnSlice,
WHEN_GATHERED.toString() );
if ( whenGathered != null )
{
-
artifactMetadata.setWhenGathered(ZonedDateTime.ofInstant(Instant.ofEpochMilli(whenGathered),
ZoneId.of("GMT")));
+
artifactMetadata.setWhenGathered(ZonedDateTime.ofInstant(Instant.ofEpochMilli(whenGathered),
STORAGE_TZ));
}
+ artifactMetadata.setChecksums(mapChecksumsReverse(getChecksums(key)));
return artifactMetadata;
}
@@ -1895,37 +1993,37 @@ public class CassandraMetadataRepository
RangeSlicesQuery<String, String, String> query = HFactory //
.createRangeSlicesQuery( keyspace, ss, ss, ss ) //
- .setColumnFamily(
cassandraArchivaManager.getArtifactMetadataFamilyName() ) //
- .setColumnNames( ArtifactMetadataModel.COLUMNS ); //
+ .setColumnFamily( cassandraArchivaManager.getChecksumFamilyName())
//
+ .setColumnNames(ARTIFACT_METADATA_MODEL_KEY); //
- query = query.addEqualsExpression( SHA1.toString(), checksum
).addEqualsExpression( REPOSITORY_NAME.toString(), repositoryId );
+ query = query.addEqualsExpression( CHECKSUM_VALUE.toString(), checksum
)
+ .addEqualsExpression( REPOSITORY_NAME.toString(), repositoryId
);
QueryResult<OrderedRows<String, String, String>> result =
query.execute();
+ List<String> artifactKeys = new ArrayList<>();
for ( Row<String, String, String> row : result.get() )
{
ColumnSlice<String, String> columnSlice = row.getColumnSlice();
- artifactMetadataMap.put( row.getKey(),
mapArtifactMetadataStringColumnSlice( columnSlice ) );
+
artifactKeys.add(columnSlice.getColumnByName(ARTIFACT_METADATA_MODEL_KEY).getValue());
}
- query = HFactory //
- .createRangeSlicesQuery( keyspace, ss, ss, ss ) //
- .setColumnFamily(
cassandraArchivaManager.getArtifactMetadataFamilyName() ) //
- .setColumnNames( NAMESPACE_ID.toString(), SIZE.toString(),
ID.toString(), FILE_LAST_MODIFIED.toString(), MD5.toString(),
PROJECT.toString(), PROJECT_VERSION.toString(),
- REPOSITORY_NAME.toString(), VERSION.toString(),
WHEN_GATHERED.toString(), SHA1.toString() ); //
-
- query = query.addEqualsExpression( MD5.toString(), checksum
).addEqualsExpression( REPOSITORY_NAME.toString(), repositoryId );
-
- result = query.execute();
-
- for ( Row<String, String, String> row : result.get() )
- {
- ColumnSlice<String, String> columnSlice = row.getColumnSlice();
+ for (String key : artifactKeys) {
+ query = HFactory //
+ .createRangeSlicesQuery(keyspace, ss, ss, ss) //
+
.setColumnFamily(cassandraArchivaManager.getArtifactMetadataFamilyName()) //
+ .setColumnNames(NAMESPACE_ID.toString(), SIZE.toString(),
ID.toString(), FILE_LAST_MODIFIED.toString(), MD5.toString(),
PROJECT.toString(), PROJECT_VERSION.toString(),
+ REPOSITORY_NAME.toString(), VERSION.toString(),
WHEN_GATHERED.toString(), SHA1.toString())
+ .setKeys(key, key);
+ result = query.execute();
- artifactMetadataMap.put( row.getKey(),
mapArtifactMetadataStringColumnSlice( columnSlice ) );
+ for (Row<String, String, String> row : result.get()) {
+ ColumnSlice<String, String> columnSlice = row.getColumnSlice();
+ artifactMetadataMap.put(row.getKey(),
mapArtifactMetadataStringColumnSlice(key, columnSlice));
+ }
}
return new ArrayList(artifactMetadataMap.values());
@@ -1993,7 +2091,8 @@ public class CassandraMetadataRepository
for ( Row<String, String, String> artifactMetadataRow :
artifactMetadataResult.get() )
{
- artifactMetadatas.add( mapArtifactMetadataStringColumnSlice(
artifactMetadataRow.getColumnSlice() ) );
+ String artifactKey = artifactMetadataRow.getKey();
+ artifactMetadatas.add( mapArtifactMetadataStringColumnSlice(
artifactKey, artifactMetadataRow.getColumnSlice() ) );
}
}
@@ -2120,13 +2219,15 @@ public class CassandraMetadataRepository
QueryResult<OrderedRows<String, String, String>> result =
query.execute();
+
+
List<ArtifactMetadata> artifactMetadatas = new ArrayList<>(
result.get().getCount() );
for ( Row<String, String, String> row : result.get() )
{
+ String key = row.getKey();
ColumnSlice<String, String> columnSlice = row.getColumnSlice();
-
- artifactMetadatas.add( mapArtifactMetadataStringColumnSlice(
columnSlice ) );
+ artifactMetadatas.add( mapArtifactMetadataStringColumnSlice( key,
columnSlice ) );
}
@@ -2240,7 +2341,8 @@ public class CassandraMetadataRepository
for ( Row<String, String, String> row : result.get() )
{
- artifactMetadatas.add( mapArtifactMetadataStringColumnSlice(
row.getColumnSlice() ) );
+ String key = row.getKey();
+ artifactMetadatas.add( mapArtifactMetadataStringColumnSlice( key,
row.getColumnSlice() ) );
}
result = HFactory.createRangeSlicesQuery( keyspace, ss, ss, ss ) //
diff --git
a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/DefaultCassandraArchivaManager.java
b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/DefaultCassandraArchivaManager.java
index 76381b6..2d8ec7d 100644
---
a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/DefaultCassandraArchivaManager.java
+++
b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/DefaultCassandraArchivaManager.java
@@ -92,6 +92,8 @@ public class DefaultCassandraArchivaManager
private String dependencyFamilyName = "dependency";
+ private String checksumFamilyName = "checksum";
+
@Value("${cassandra.host}")
private String cassandraHost;
@@ -388,6 +390,49 @@ public class DefaultCassandraArchivaManager
}
+ // Checksum table
+ {
+ final ColumnFamilyDefinition checksumCf =
+ HFactory.createColumnFamilyDefinition(
keyspace.getKeyspaceName(), //
+ getChecksumFamilyName(), //
+ ComparatorType.UTF8TYPE );
+
+ BasicColumnDefinition artifactMetatadaModel_key = new
BasicColumnDefinition();
+ artifactMetatadaModel_key.setName(
StringSerializer.get().toByteBuffer( "artifactMetadataModel.key" ) );
+ artifactMetatadaModel_key.setIndexName(
"artifactMetadataModel_key" );
+ artifactMetatadaModel_key.setIndexType( ColumnIndexType.KEYS );
+ artifactMetatadaModel_key.setValidationClass(
ComparatorType.UTF8TYPE.getClassName() );
+ checksumCf.addColumnDefinition( artifactMetatadaModel_key );
+
+
+ BasicColumnDefinition checksumAlgorithmColumn = new
BasicColumnDefinition();
+ checksumAlgorithmColumn.setName(
StringSerializer.get().toByteBuffer( CHECKSUM_ALG.toString() ) );
+ checksumAlgorithmColumn.setIndexName( CHECKSUM_ALG.toString() );
+ checksumAlgorithmColumn.setIndexType( ColumnIndexType.KEYS );
+ checksumAlgorithmColumn.setValidationClass(
ComparatorType.UTF8TYPE.getClassName() );
+ checksumCf.addColumnDefinition( checksumAlgorithmColumn );
+
+ BasicColumnDefinition checksumValueColumn = new
BasicColumnDefinition();
+ checksumValueColumn.setName( StringSerializer.get().toByteBuffer(
CHECKSUM_VALUE.toString() ) );
+ checksumValueColumn.setIndexName( CHECKSUM_VALUE.toString() );
+ checksumValueColumn.setIndexType( ColumnIndexType.KEYS );
+ checksumValueColumn.setValidationClass(
ComparatorType.UTF8TYPE.getClassName() );
+ checksumCf.addColumnDefinition( checksumValueColumn );
+
+ BasicColumnDefinition repositoryNameColumn = new
BasicColumnDefinition();
+ repositoryNameColumn.setName( StringSerializer.get().toByteBuffer(
REPOSITORY_NAME.toString() ) );
+ repositoryNameColumn.setIndexName( REPOSITORY_NAME.toString() );
+ repositoryNameColumn.setIndexType( ColumnIndexType.KEYS );
+ repositoryNameColumn.setValidationClass(
ComparatorType.UTF8TYPE.getClassName() );
+ checksumCf.addColumnDefinition( repositoryNameColumn );
+
+
+ cfds.add( checksumCf );
+
+ // creating indexes for cql query
+
+ }
+
// mailinglist table
{
final ColumnFamilyDefinition mailingListCf =
@@ -553,4 +598,9 @@ public class DefaultCassandraArchivaManager
{
return dependencyFamilyName;
}
+
+ @Override
+ public String getChecksumFamilyName() {
+ return checksumFamilyName;
+ }
}
diff --git
a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/model/ArtifactMetadataModel.java
b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/model/ArtifactMetadataModel.java
index db04997..64ba9a3 100644
---
a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/model/ArtifactMetadataModel.java
+++
b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/model/ArtifactMetadataModel.java
@@ -23,6 +23,8 @@ import
org.apache.archiva.metadata.repository.cassandra.CassandraUtils;
import java.io.Serializable;
import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
import static
org.apache.archiva.metadata.repository.cassandra.model.ColumnNames.*;
@@ -62,6 +64,8 @@ public class ArtifactMetadataModel
private long whenGathered;
+ private Map<String, String> checksums = new HashMap<>();
+
public ArtifactMetadataModel()
{
// no op
@@ -195,6 +199,22 @@ public class ArtifactMetadataModel
this.whenGathered = whenGathered;
}
+ public void setChecksum(String type, String value) {
+ this.checksums.put(type, value);
+ }
+
+ public String getChecksum(String type) {
+ return this.checksums.get(type);
+ }
+
+ public void setChecksums(Map<String,String> checksums) {
+ this.checksums = checksums;
+ }
+
+ public Map<String,String> getChecksums() {
+ return this.checksums;
+ }
+
@Override
public String toString()
diff --git
a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/model/ColumnNames.java
b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/model/ColumnNames.java
index fa81935..8ea2f38 100644
---
a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/model/ColumnNames.java
+++
b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/model/ColumnNames.java
@@ -45,7 +45,9 @@ public enum ColumnNames
ARTIFACT_ID( "artifactId" ),
DESCRIPTION( "description" ),
URL( "url" ),
- WHEN_GATHERED( "whenGathered" );
+ WHEN_GATHERED( "whenGathered" ),
+ CHECKSUM_ALG("checksumAlgorithm"),
+ CHECKSUM_VALUE("checksumValue");
private final String name;
diff --git
a/archiva-modules/plugins/metadata-store-cassandra/src/test/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepositoryTest.java
b/archiva-modules/plugins/metadata-store-cassandra/src/test/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepositoryTest.java
index b395e36..b095f05 100644
---
a/archiva-modules/plugins/metadata-store-cassandra/src/test/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepositoryTest.java
+++
b/archiva-modules/plugins/metadata-store-cassandra/src/test/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepositoryTest.java
@@ -38,7 +38,6 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
-import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
@@ -79,7 +78,10 @@ public class CassandraMetadataRepositoryTest
public void setUp()
throws Exception
{
+
super.setUp();
+ assertMaxTries =1;
+ assertRetrySleepMs=10;
Path directory = Paths.get( "target/test-repositories" );
if ( Files.exists(directory) )
