Author: olamy
Date: Wed Nov 20 03:15:01 2013
New Revision: 1543703
URL: http://svn.apache.org/r1543703
Log:
start some perf improvement for cassandra
Modified:
archiva/trunk/archiva-modules/plugins/metadata-store-cassandra/pom.xml
archiva/trunk/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepository.java
archiva/trunk/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/DefaultCassandraEntityManagerFactory.java
archiva/trunk/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/model/Namespace.java
archiva/trunk/archiva-modules/plugins/pom.xml
Modified: archiva/trunk/archiva-modules/plugins/metadata-store-cassandra/pom.xml
URL:
http://svn.apache.org/viewvc/archiva/trunk/archiva-modules/plugins/metadata-store-cassandra/pom.xml?rev=1543703&r1=1543702&r2=1543703&view=diff
==============================================================================
--- archiva/trunk/archiva-modules/plugins/metadata-store-cassandra/pom.xml
(original)
+++ archiva/trunk/archiva-modules/plugins/metadata-store-cassandra/pom.xml Wed
Nov 20 03:15:01 2013
@@ -256,8 +256,8 @@
org.springframework.context,
org.springframework.stereotype,
com.netflix.astyanax.serializers,
- org.apache.commons.codec.binary,
- org.apache.commons.io
+ org.apache.commons.codec.binary,
+ org.apache.commons.io
</Import-Package>
</instructions>
</configuration>
Modified:
archiva/trunk/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepository.java
URL:
http://svn.apache.org/viewvc/archiva/trunk/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepository.java?rev=1543703&r1=1543702&r2=1543703&view=diff
==============================================================================
---
archiva/trunk/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepository.java
(original)
+++
archiva/trunk/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepository.java
Wed Nov 20 03:15:01 2013
@@ -316,15 +316,21 @@ public class CassandraMetadataRepository
try
{
- //final List<Namespace> namespaceList =
- // getNamespaceEntityManager().find( "SELECT name FROM
namespace WHERE repository.id='"+ repoId + "'" );
- //final Set<String> namespaces = new HashSet<String>(
namespaceList.size() );
+ RootNamesSpaceVisitAll rootNamesSpaceVisitAll = new
RootNamesSpaceVisitAll( repoId );
- final Set<String> namespaces = new HashSet<String>( );
+ getNamespaceEntityManager().visitAll( rootNamesSpaceVisitAll );
+ return rootNamesSpaceVisitAll.namespaces;
+
+
+ // using cql query with index
/*
- for ( Namespace namespace : namespaceList )
+ List<Namespace> namespacesList = getNamespaceEntityManager().find(
"SELECT * from namespace where id <> null AND repositoryid = '" + repoId + "'"
);
+
+ Set<String> namespaces = new HashSet<String>();
+
+ for (Namespace namespace : namespacesList)
{
String name = namespace.getName();
if ( StringUtils.isNotEmpty( name ) )
@@ -332,83 +338,109 @@ public class CassandraMetadataRepository
namespaces.add( StringUtils.substringBefore( name, "." ) );
}
}
+
+ return namespaces;
*/
+ }
+ catch ( Exception e )
+ {
+ throw new MetadataResolutionException( e.getMessage(), e );
+ }
+ }
- getNamespaceEntityManager().visitAll( new Function<Namespace,
Boolean>()
+ private static class RootNamesSpaceVisitAll
+ implements Function<Namespace, Boolean>
+ {
+ private String repoId;
+
+ Set<String> namespaces = new HashSet<String>();
+
+ private RootNamesSpaceVisitAll( String repoId )
+ {
+ this.repoId = repoId;
+ }
+
+ // @Nullable add dependency ?
+ @Override
+ public Boolean apply( Namespace namespace )
+ {
+ if ( namespace != null && namespace.getRepository() != null &&
StringUtils.equalsIgnoreCase( repoId,
+
namespace.getRepository().getId() ) )
{
- // @Nullable add dependency ?
- @Override
- public Boolean apply( Namespace namespace )
+ String name = namespace.getName();
+ if ( StringUtils.isNotEmpty( name ) )
{
- if ( namespace != null && namespace.getRepository() != null
- && StringUtils.equalsIgnoreCase( repoId,
namespace.getRepository().getId() ) )
- {
- String name = namespace.getName();
- if ( StringUtils.isNotEmpty( name ) )
- {
- namespaces.add( StringUtils.substringBefore( name,
"." ) );
- }
- }
- return Boolean.TRUE;
+ namespaces.add( StringUtils.substringBefore( name, "." ) );
}
- } );
+ }
+ return Boolean.TRUE;
+ }
+ }
- return namespaces;
+ @Override
+ public Collection<String> getNamespaces( final String repoId, final String
namespaceId )
+ throws MetadataResolutionException
+ {
+ try
+ {
+ final FindNamesSpaceVisitAll findNamesSpaceVisitAll = new
FindNamesSpaceVisitAll( repoId, namespaceId );
+
+ getNamespaceEntityManager().visitAll( findNamesSpaceVisitAll );
+
+ return findNamesSpaceVisitAll.namespaces;
}
catch ( PersistenceException e )
{
throw new MetadataResolutionException( e.getMessage(), e );
}
+
}
- @Override
- public Collection<String> getNamespaces( final String repoId, final String
namespaceId )
- throws MetadataResolutionException
+ private static class FindNamesSpaceVisitAll
+ implements Function<Namespace, Boolean>
{
- try
+ private String repoId;
+
+ private String namespaceId;
+
+ Set<String> namespaces = new HashSet<String>();
+
+ private FindNamesSpaceVisitAll( String repoId, String namespaceId )
{
- final Set<String> namespaces = new HashSet<String>();
+ this.repoId = repoId;
+ this.namespaceId = namespaceId;
+ }
- getNamespaceEntityManager().visitAll( new Function<Namespace,
Boolean>()
+ // @Nullable add dependency ?
+ @Override
+ public Boolean apply( Namespace namespace )
+ {
+ if ( namespace != null && namespace.getRepository() != null &&
StringUtils.equalsIgnoreCase( repoId,
+
namespace.getRepository().getId() ) )
{
- // @Nullable add dependency ?
- @Override
- public Boolean apply( Namespace namespace )
+ String currentNamespace = namespace.getName();
+ // we only return childs
+ if ( StringUtils.startsWith( currentNamespace, namespaceId )
&& (
+ StringUtils.length( currentNamespace ) >
StringUtils.length( namespaceId ) ) )
{
- if ( namespace != null && namespace.getRepository() !=
null && StringUtils.equalsIgnoreCase( repoId,
-
namespace.getRepository().getId() ) )
- {
- String currentNamespace = namespace.getName();
- // we only return childs
- if ( StringUtils.startsWith( currentNamespace,
namespaceId ) && (
- StringUtils.length( currentNamespace ) >
StringUtils.length( namespaceId ) ) )
- {
- // store after namespaceId '.' but before next '.'
- // call org namespace org.apache.maven.shared ->
stored apache
+ // store after namespaceId '.' but before next '.'
+ // call org namespace org.apache.maven.shared -> stored
apache
- String calledNamespace =
- StringUtils.endsWith( namespaceId, "." ) ?
namespaceId : namespaceId + ".";
- String storedNamespace =
StringUtils.substringAfter( currentNamespace, calledNamespace );
+ String calledNamespace =
+ StringUtils.endsWith( namespaceId, "." ) ? namespaceId
: namespaceId + ".";
+ String storedNamespace = StringUtils.substringAfter(
currentNamespace, calledNamespace );
- storedNamespace = StringUtils.substringBefore(
storedNamespace, "." );
+ storedNamespace = StringUtils.substringBefore(
storedNamespace, "." );
- namespaces.add( storedNamespace );
- }
- }
- return Boolean.TRUE;
+ namespaces.add( storedNamespace );
}
- } );
-
- return namespaces;
- }
- catch ( PersistenceException e )
- {
- throw new MetadataResolutionException( e.getMessage(), e );
+ }
+ return Boolean.TRUE;
}
-
}
+
public List<String> getNamespaces( final String repoId )
throws MetadataResolutionException
{
Modified:
archiva/trunk/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/DefaultCassandraEntityManagerFactory.java
URL:
http://svn.apache.org/viewvc/archiva/trunk/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/DefaultCassandraEntityManagerFactory.java?rev=1543703&r1=1543702&r2=1543703&view=diff
==============================================================================
---
archiva/trunk/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/DefaultCassandraEntityManagerFactory.java
(original)
+++
archiva/trunk/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/DefaultCassandraEntityManagerFactory.java
Wed Nov 20 03:15:01 2013
@@ -28,7 +28,10 @@ import com.netflix.astyanax.connectionpo
import
com.netflix.astyanax.connectionpool.impl.ConnectionPoolConfigurationImpl;
import com.netflix.astyanax.connectionpool.impl.ConnectionPoolType;
import com.netflix.astyanax.connectionpool.impl.Slf4jConnectionPoolMonitorImpl;
+import com.netflix.astyanax.ddl.ColumnDefinition;
+import com.netflix.astyanax.ddl.ColumnFamilyDefinition;
import com.netflix.astyanax.ddl.KeyspaceDefinition;
+import com.netflix.astyanax.entitystore.CompositeEntityManager;
import com.netflix.astyanax.entitystore.DefaultEntityManager;
import com.netflix.astyanax.entitystore.EntityManager;
import com.netflix.astyanax.impl.AstyanaxConfigurationImpl;
@@ -48,6 +51,7 @@ import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import javax.inject.Inject;
import javax.persistence.PersistenceException;
+import java.util.Map;
import java.util.Properties;
/**
@@ -95,14 +99,22 @@ public class DefaultCassandraEntityManag
{
String cassandraHost = System.getProperty( "cassandraHost",
"localhost" );
String cassandraPort = System.getProperty( "cassandraPort" );
+ String cqlVersion = System.getProperty( "cassandra.cqlversion",
"3.0.0" );
keyspaceContext = new AstyanaxContext.Builder().forCluster(
CLUSTER_NAME ).forKeyspace(
KEYSPACE_NAME ).withAstyanaxConfiguration(
- new AstyanaxConfigurationImpl().setDiscoveryType(
NodeDiscoveryType.RING_DESCRIBE ).setConnectionPoolType(
- ConnectionPoolType.TOKEN_AWARE )
).withConnectionPoolConfiguration(
- new ConnectionPoolConfigurationImpl( CLUSTER_NAME + "_" +
KEYSPACE_NAME ).setSocketTimeout(
- 30000 ).setMaxTimeoutWhenExhausted( 2000 ).setMaxConnsPerHost(
20 ).setInitConnsPerHost( 10 ).setSeeds(
- cassandraHost + ":" + cassandraPort )
).withConnectionPoolMonitor(
- new Slf4jConnectionPoolMonitorImpl() ).buildKeyspace(
ThriftFamilyFactory.getInstance() );
+ new AstyanaxConfigurationImpl()
+ //.setCqlVersion( cqlVersion )
+ .setDiscoveryType( NodeDiscoveryType.RING_DESCRIBE )
+ .setConnectionPoolType( ConnectionPoolType.TOKEN_AWARE ) )
+ .withConnectionPoolConfiguration(
+ new ConnectionPoolConfigurationImpl( CLUSTER_NAME + "_" +
KEYSPACE_NAME )
+ .setSocketTimeout( 30000 )
+ .setMaxTimeoutWhenExhausted( 2000 )
+ .setMaxConnsPerHost( 20 )
+ .setInitConnsPerHost( 10 )
+ .setSeeds( cassandraHost + ":" + cassandraPort ) )
+ .withConnectionPoolMonitor( new Slf4jConnectionPoolMonitorImpl() )
+ .buildKeyspace( ThriftFamilyFactory.getInstance() );
this.start();
@@ -149,8 +161,12 @@ public class DefaultCassandraEntityManag
try
{
repositoryEntityManager =
- new DefaultEntityManager.Builder<Repository,
String>().withEntityType( Repository.class ).withKeyspace(
- keyspace ).withAutoCommit( true ).build();
+ DefaultEntityManager.<Repository, String>builder()
+ .withEntityType( Repository.class )
+ .withKeyspace( keyspace )
+ .withAutoCommit( true )
+ .withColumnFamily( "repository" )
+ .build();
boolean exists = columnFamilyExists( "repository" );
// TODO very basic test we must test model change too
if ( !exists )
@@ -159,18 +175,38 @@ public class DefaultCassandraEntityManag
}
namespaceEntityManager =
- new DefaultEntityManager.Builder<Namespace,
String>().withEntityType( Namespace.class ).withKeyspace(
- keyspace ).withAutoCommit( true ).build();
+ DefaultEntityManager.<Namespace, String>builder()
+ .withEntityType( Namespace.class )
+ .withKeyspace( keyspace )
+ .withAutoCommit( true )
+ .withColumnFamily( "namespace" )
+ .build();
exists = columnFamilyExists( "namespace" );
if ( !exists )
{
- namespaceEntityManager.createStorage( null );
+ // create secondary index
+
+ options =
+ ImmutableMap.<String, Object>builder()
+ .put("repositoryid", ImmutableMap.<String,
Object>builder()
+ .put("validation_class", "UTF8Type")
+ .put("index_name", "Indexrepositoryid")
+ .put("index_type", "KEYS")
+ .build()).build();
+
+ namespaceEntityManager.createStorage( options );
}
+
+
projectEntityManager =
- new DefaultEntityManager.Builder<Project,
String>().withEntityType( Project.class ).withKeyspace(
- keyspace ).withAutoCommit( true ).build();
+ DefaultEntityManager.<Project, String>builder()
+ .withEntityType( Project.class )
+ .withKeyspace( keyspace )
+ .withAutoCommit( true )
+ .withColumnFamily( "project" )
+ .build();
exists = columnFamilyExists( "project" );
if ( !exists )
@@ -179,8 +215,12 @@ public class DefaultCassandraEntityManag
}
artifactMetadataModelEntityManager =
- new DefaultEntityManager.Builder<ArtifactMetadataModel,
String>().withEntityType(
- ArtifactMetadataModel.class ).withAutoCommit( true
).withKeyspace( keyspace ).build();
+ DefaultEntityManager.<ArtifactMetadataModel, String>builder()
+ .withEntityType( ArtifactMetadataModel.class )
+ .withAutoCommit( true )
+ .withKeyspace( keyspace )
+ .withColumnFamily( "artifactmetadatamodel" )
+ .build();
exists = columnFamilyExists( "artifactmetadatamodel" );
if ( !exists )
@@ -189,8 +229,12 @@ public class DefaultCassandraEntityManag
}
metadataFacetModelEntityManager =
- new DefaultEntityManager.Builder<MetadataFacetModel,
String>().withEntityType(
- MetadataFacetModel.class ).withAutoCommit( true
).withKeyspace( keyspace ).build();
+ DefaultEntityManager.<MetadataFacetModel, String>builder()
+ .withEntityType( MetadataFacetModel.class )
+ .withAutoCommit( true )
+ .withKeyspace( keyspace )
+ .withColumnFamily( "metadatafacetmodel" )
+ .build();
exists = columnFamilyExists( "metadatafacetmodel" );
if ( !exists )
@@ -199,8 +243,12 @@ public class DefaultCassandraEntityManag
}
projectVersionMetadataModelEntityManager =
- new DefaultEntityManager.Builder<ProjectVersionMetadataModel,
String>().withEntityType(
- ProjectVersionMetadataModel.class ).withAutoCommit( true
).withKeyspace( keyspace ).build();
+ DefaultEntityManager.<ProjectVersionMetadataModel,
String>builder()
+ .withEntityType( ProjectVersionMetadataModel.class )
+ .withAutoCommit( true )
+ .withKeyspace( keyspace )
+ .withColumnFamily( "projectversionmetadatamodel" )
+ .build();
exists = columnFamilyExists( "projectversionmetadatamodel" );
if ( !exists )
Modified:
archiva/trunk/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/model/Namespace.java
URL:
http://svn.apache.org/viewvc/archiva/trunk/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/model/Namespace.java?rev=1543703&r1=1543702&r2=1543703&view=diff
==============================================================================
---
archiva/trunk/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/model/Namespace.java
(original)
+++
archiva/trunk/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/model/Namespace.java
Wed Nov 20 03:15:01 2013
@@ -38,7 +38,6 @@ public class Namespace
{
@Id
- @Column( name = "id" )
@Serializer( DeflateStringSerializer.class )
private String id;
@@ -46,8 +45,13 @@ public class Namespace
@Serializer( DeflateStringSerializer.class )
private String name;
- @Column( name = "repository" )
- private Repository repository;
+ @Column( name = "repositoryid" )
+ @Serializer( DeflateStringSerializer.class )
+ private String repositoryId;
+
+ @Column( name = "repositoryname" )
+ @Serializer( DeflateStringSerializer.class )
+ private String repositoryName;
//@ManyToOne(cascade = { CascadeType.ALL }, fetch = FetchType.EAGER)
//@JoinColumn(name = "repository_id")
@@ -64,7 +68,8 @@ public class Namespace
{
this.id = new KeyBuilder().withNamespace( id ).withRepositoryId(
repository.getId() ).build();
this.name = id;
- this.repository = repository;
+ this.repositoryId = repository.getId();
+ this.repositoryName = repository.getName();
}
public String getId()
@@ -89,12 +94,13 @@ public class Namespace
public Repository getRepository()
{
- return repository;
+ return new Repository(this.repositoryId);
}
public void setRepository( Repository repository )
{
- this.repository = repository;
+ this.repositoryId = repository.getId();
+ this.repositoryName = repository.getName();
}
@Override
@@ -115,7 +121,7 @@ public class Namespace
{
return false;
}
- if ( !repository.getId().equals( namespace.repository.getId() ) )
+ if ( !repositoryId.equals( namespace.repositoryId ) )
{
return false;
}
@@ -127,7 +133,7 @@ public class Namespace
public int hashCode()
{
int result = id.hashCode();
- result = 31 * result + repository.getId().hashCode();
+ result = 31 * result + repositoryId.hashCode();
return result;
}
@@ -137,7 +143,7 @@ public class Namespace
final StringBuilder sb = new StringBuilder( "Namespace{" );
sb.append( "id='" ).append( id ).append( '\'' );
sb.append( ", name='" ).append( name ).append( '\'' );
- sb.append( ", repository='" ).append( repository ).append( '\'' );
+ sb.append( ", repository='" ).append( repositoryId ).append( '\'' );
sb.append( '}' );
return sb.toString();
}
Modified: archiva/trunk/archiva-modules/plugins/pom.xml
URL:
http://svn.apache.org/viewvc/archiva/trunk/archiva-modules/plugins/pom.xml?rev=1543703&r1=1543702&r2=1543703&view=diff
==============================================================================
--- archiva/trunk/archiva-modules/plugins/pom.xml (original)
+++ archiva/trunk/archiva-modules/plugins/pom.xml Wed Nov 20 03:15:01 2013
@@ -39,6 +39,6 @@
<module>stage-repository-merge</module>
<module>generic-metadata-support</module>
<module>metadata-store-jcr</module>
- <module>metadata-store-cassandra</module>
</modules>
+
</project>
