From 0d911a208a264436aff0882b2870a8846b036681 Mon Sep 17 00:00:00 2001 From: Olivier Lamy Date: Wed, 20 Nov 2013 03:15:01 +0000 Subject: [PATCH] start some perf improvement for cassandra git-svn-id: https://svn.apache.org/repos/asf/archiva/trunk@1543703 13f79535-47bb-0310-9956-ffa450edef68 --- .../plugins/metadata-store-cassandra/pom.xml | 4 +- .../CassandraMetadataRepository.java | 144 +++++++++++------- .../DefaultCassandraEntityManagerFactory.java | 86 ++++++++--- .../repository/cassandra/model/Namespace.java | 24 +-- archiva-modules/plugins/pom.xml | 2 +- 5 files changed, 173 insertions(+), 87 deletions(-) diff --git a/archiva-modules/plugins/metadata-store-cassandra/pom.xml b/archiva-modules/plugins/metadata-store-cassandra/pom.xml index 323c57a33..f7b17c018 100644 --- a/archiva-modules/plugins/metadata-store-cassandra/pom.xml +++ b/archiva-modules/plugins/metadata-store-cassandra/pom.xml @@ -256,8 +256,8 @@ org.springframework.context, org.springframework.stereotype, com.netflix.astyanax.serializers, - org.apache.commons.codec.binary, - org.apache.commons.io + org.apache.commons.codec.binary, + org.apache.commons.io diff --git a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepository.java b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepository.java index 7c98de0fa..6ab95bba0 100644 --- a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepository.java +++ b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepository.java @@ -316,15 +316,21 @@ public class CassandraMetadataRepository try { - //final List namespaceList = - // getNamespaceEntityManager().find( "SELECT name FROM namespace WHERE repository.id='"+ repoId + "'" ); - //final Set namespaces = new HashSet( namespaceList.size() ); + RootNamesSpaceVisitAll rootNamesSpaceVisitAll = new RootNamesSpaceVisitAll( repoId ); - final Set namespaces = new HashSet( ); + getNamespaceEntityManager().visitAll( rootNamesSpaceVisitAll ); + return rootNamesSpaceVisitAll.namespaces; + + + // using cql query with index /* - for ( Namespace namespace : namespaceList ) + List namespacesList = getNamespaceEntityManager().find( "SELECT * from namespace where id <> null AND repositoryid = '" + repoId + "'" ); + + Set namespaces = new HashSet(); + + for (Namespace namespace : namespacesList) { String name = namespace.getName(); if ( StringUtils.isNotEmpty( name ) ) @@ -332,83 +338,109 @@ public class CassandraMetadataRepository namespaces.add( StringUtils.substringBefore( name, "." ) ); } } + + return namespaces; */ + } + catch ( Exception e ) + { + throw new MetadataResolutionException( e.getMessage(), e ); + } + } - getNamespaceEntityManager().visitAll( new Function() + private static class RootNamesSpaceVisitAll + implements Function + { + private String repoId; + + Set namespaces = new HashSet(); + + private RootNamesSpaceVisitAll( String repoId ) + { + this.repoId = repoId; + } + + // @Nullable add dependency ? + @Override + public Boolean apply( Namespace namespace ) + { + if ( namespace != null && namespace.getRepository() != null && StringUtils.equalsIgnoreCase( repoId, + namespace.getRepository().getId() ) ) { - // @Nullable add dependency ? - @Override - public Boolean apply( Namespace namespace ) + String name = namespace.getName(); + if ( StringUtils.isNotEmpty( name ) ) { - if ( namespace != null && namespace.getRepository() != null - && StringUtils.equalsIgnoreCase( repoId, namespace.getRepository().getId() ) ) - { - String name = namespace.getName(); - if ( StringUtils.isNotEmpty( name ) ) - { - namespaces.add( StringUtils.substringBefore( name, "." ) ); - } - } - return Boolean.TRUE; + namespaces.add( StringUtils.substringBefore( name, "." ) ); } - } ); + } + return Boolean.TRUE; + } + } - return namespaces; + @Override + public Collection getNamespaces( final String repoId, final String namespaceId ) + throws MetadataResolutionException + { + try + { + final FindNamesSpaceVisitAll findNamesSpaceVisitAll = new FindNamesSpaceVisitAll( repoId, namespaceId ); + + getNamespaceEntityManager().visitAll( findNamesSpaceVisitAll ); + + return findNamesSpaceVisitAll.namespaces; } catch ( PersistenceException e ) { throw new MetadataResolutionException( e.getMessage(), e ); } + } - @Override - public Collection getNamespaces( final String repoId, final String namespaceId ) - throws MetadataResolutionException + private static class FindNamesSpaceVisitAll + implements Function { - try + private String repoId; + + private String namespaceId; + + Set namespaces = new HashSet(); + + private FindNamesSpaceVisitAll( String repoId, String namespaceId ) { - final Set namespaces = new HashSet(); + this.repoId = repoId; + this.namespaceId = namespaceId; + } - getNamespaceEntityManager().visitAll( new Function() + // @Nullable add dependency ? + @Override + public Boolean apply( Namespace namespace ) + { + if ( namespace != null && namespace.getRepository() != null && StringUtils.equalsIgnoreCase( repoId, + namespace.getRepository().getId() ) ) { - // @Nullable add dependency ? - @Override - public Boolean apply( Namespace namespace ) + String currentNamespace = namespace.getName(); + // we only return childs + if ( StringUtils.startsWith( currentNamespace, namespaceId ) && ( + StringUtils.length( currentNamespace ) > StringUtils.length( namespaceId ) ) ) { - if ( namespace != null && namespace.getRepository() != null && StringUtils.equalsIgnoreCase( repoId, - namespace.getRepository().getId() ) ) - { - String currentNamespace = namespace.getName(); - // we only return childs - if ( StringUtils.startsWith( currentNamespace, namespaceId ) && ( - StringUtils.length( currentNamespace ) > StringUtils.length( namespaceId ) ) ) - { - // store after namespaceId '.' but before next '.' - // call org namespace org.apache.maven.shared -> stored apache + // store after namespaceId '.' but before next '.' + // call org namespace org.apache.maven.shared -> stored apache - String calledNamespace = - StringUtils.endsWith( namespaceId, "." ) ? namespaceId : namespaceId + "."; - String storedNamespace = StringUtils.substringAfter( currentNamespace, calledNamespace ); + String calledNamespace = + StringUtils.endsWith( namespaceId, "." ) ? namespaceId : namespaceId + "."; + String storedNamespace = StringUtils.substringAfter( currentNamespace, calledNamespace ); - storedNamespace = StringUtils.substringBefore( storedNamespace, "." ); + storedNamespace = StringUtils.substringBefore( storedNamespace, "." ); - namespaces.add( storedNamespace ); - } - } - return Boolean.TRUE; + namespaces.add( storedNamespace ); } - } ); - - return namespaces; - } - catch ( PersistenceException e ) - { - throw new MetadataResolutionException( e.getMessage(), e ); + } + return Boolean.TRUE; } - } + public List getNamespaces( final String repoId ) throws MetadataResolutionException { diff --git a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/DefaultCassandraEntityManagerFactory.java b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/DefaultCassandraEntityManagerFactory.java index 8ebfc2d29..24150e6ef 100644 --- a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/DefaultCassandraEntityManagerFactory.java +++ b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/DefaultCassandraEntityManagerFactory.java @@ -28,7 +28,10 @@ import com.netflix.astyanax.connectionpool.exceptions.NotFoundException; import com.netflix.astyanax.connectionpool.impl.ConnectionPoolConfigurationImpl; import com.netflix.astyanax.connectionpool.impl.ConnectionPoolType; import com.netflix.astyanax.connectionpool.impl.Slf4jConnectionPoolMonitorImpl; +import com.netflix.astyanax.ddl.ColumnDefinition; +import com.netflix.astyanax.ddl.ColumnFamilyDefinition; import com.netflix.astyanax.ddl.KeyspaceDefinition; +import com.netflix.astyanax.entitystore.CompositeEntityManager; import com.netflix.astyanax.entitystore.DefaultEntityManager; import com.netflix.astyanax.entitystore.EntityManager; import com.netflix.astyanax.impl.AstyanaxConfigurationImpl; @@ -48,6 +51,7 @@ import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import javax.inject.Inject; import javax.persistence.PersistenceException; +import java.util.Map; import java.util.Properties; /** @@ -95,14 +99,22 @@ public class DefaultCassandraEntityManagerFactory { String cassandraHost = System.getProperty( "cassandraHost", "localhost" ); String cassandraPort = System.getProperty( "cassandraPort" ); + String cqlVersion = System.getProperty( "cassandra.cqlversion", "3.0.0" ); keyspaceContext = new AstyanaxContext.Builder().forCluster( CLUSTER_NAME ).forKeyspace( KEYSPACE_NAME ).withAstyanaxConfiguration( - new AstyanaxConfigurationImpl().setDiscoveryType( NodeDiscoveryType.RING_DESCRIBE ).setConnectionPoolType( - ConnectionPoolType.TOKEN_AWARE ) ).withConnectionPoolConfiguration( - new ConnectionPoolConfigurationImpl( CLUSTER_NAME + "_" + KEYSPACE_NAME ).setSocketTimeout( - 30000 ).setMaxTimeoutWhenExhausted( 2000 ).setMaxConnsPerHost( 20 ).setInitConnsPerHost( 10 ).setSeeds( - cassandraHost + ":" + cassandraPort ) ).withConnectionPoolMonitor( - new Slf4jConnectionPoolMonitorImpl() ).buildKeyspace( ThriftFamilyFactory.getInstance() ); + new AstyanaxConfigurationImpl() + //.setCqlVersion( cqlVersion ) + .setDiscoveryType( NodeDiscoveryType.RING_DESCRIBE ) + .setConnectionPoolType( ConnectionPoolType.TOKEN_AWARE ) ) + .withConnectionPoolConfiguration( + new ConnectionPoolConfigurationImpl( CLUSTER_NAME + "_" + KEYSPACE_NAME ) + .setSocketTimeout( 30000 ) + .setMaxTimeoutWhenExhausted( 2000 ) + .setMaxConnsPerHost( 20 ) + .setInitConnsPerHost( 10 ) + .setSeeds( cassandraHost + ":" + cassandraPort ) ) + .withConnectionPoolMonitor( new Slf4jConnectionPoolMonitorImpl() ) + .buildKeyspace( ThriftFamilyFactory.getInstance() ); this.start(); @@ -149,8 +161,12 @@ public class DefaultCassandraEntityManagerFactory try { repositoryEntityManager = - new DefaultEntityManager.Builder().withEntityType( Repository.class ).withKeyspace( - keyspace ).withAutoCommit( true ).build(); + DefaultEntityManager.builder() + .withEntityType( Repository.class ) + .withKeyspace( keyspace ) + .withAutoCommit( true ) + .withColumnFamily( "repository" ) + .build(); boolean exists = columnFamilyExists( "repository" ); // TODO very basic test we must test model change too if ( !exists ) @@ -159,18 +175,38 @@ public class DefaultCassandraEntityManagerFactory } namespaceEntityManager = - new DefaultEntityManager.Builder().withEntityType( Namespace.class ).withKeyspace( - keyspace ).withAutoCommit( true ).build(); + DefaultEntityManager.builder() + .withEntityType( Namespace.class ) + .withKeyspace( keyspace ) + .withAutoCommit( true ) + .withColumnFamily( "namespace" ) + .build(); exists = columnFamilyExists( "namespace" ); if ( !exists ) { - namespaceEntityManager.createStorage( null ); + // create secondary index + + options = + ImmutableMap.builder() + .put("repositoryid", ImmutableMap.builder() + .put("validation_class", "UTF8Type") + .put("index_name", "Indexrepositoryid") + .put("index_type", "KEYS") + .build()).build(); + + namespaceEntityManager.createStorage( options ); } + + projectEntityManager = - new DefaultEntityManager.Builder().withEntityType( Project.class ).withKeyspace( - keyspace ).withAutoCommit( true ).build(); + DefaultEntityManager.builder() + .withEntityType( Project.class ) + .withKeyspace( keyspace ) + .withAutoCommit( true ) + .withColumnFamily( "project" ) + .build(); exists = columnFamilyExists( "project" ); if ( !exists ) @@ -179,8 +215,12 @@ public class DefaultCassandraEntityManagerFactory } artifactMetadataModelEntityManager = - new DefaultEntityManager.Builder().withEntityType( - ArtifactMetadataModel.class ).withAutoCommit( true ).withKeyspace( keyspace ).build(); + DefaultEntityManager.builder() + .withEntityType( ArtifactMetadataModel.class ) + .withAutoCommit( true ) + .withKeyspace( keyspace ) + .withColumnFamily( "artifactmetadatamodel" ) + .build(); exists = columnFamilyExists( "artifactmetadatamodel" ); if ( !exists ) @@ -189,8 +229,12 @@ public class DefaultCassandraEntityManagerFactory } metadataFacetModelEntityManager = - new DefaultEntityManager.Builder().withEntityType( - MetadataFacetModel.class ).withAutoCommit( true ).withKeyspace( keyspace ).build(); + DefaultEntityManager.builder() + .withEntityType( MetadataFacetModel.class ) + .withAutoCommit( true ) + .withKeyspace( keyspace ) + .withColumnFamily( "metadatafacetmodel" ) + .build(); exists = columnFamilyExists( "metadatafacetmodel" ); if ( !exists ) @@ -199,8 +243,12 @@ public class DefaultCassandraEntityManagerFactory } projectVersionMetadataModelEntityManager = - new DefaultEntityManager.Builder().withEntityType( - ProjectVersionMetadataModel.class ).withAutoCommit( true ).withKeyspace( keyspace ).build(); + DefaultEntityManager.builder() + .withEntityType( ProjectVersionMetadataModel.class ) + .withAutoCommit( true ) + .withKeyspace( keyspace ) + .withColumnFamily( "projectversionmetadatamodel" ) + .build(); exists = columnFamilyExists( "projectversionmetadatamodel" ); if ( !exists ) diff --git a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/model/Namespace.java b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/model/Namespace.java index 14da6fffb..8c2ff27c3 100644 --- a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/model/Namespace.java +++ b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/model/Namespace.java @@ -38,7 +38,6 @@ public class Namespace { @Id - @Column( name = "id" ) @Serializer( DeflateStringSerializer.class ) private String id; @@ -46,8 +45,13 @@ public class Namespace @Serializer( DeflateStringSerializer.class ) private String name; - @Column( name = "repository" ) - private Repository repository; + @Column( name = "repositoryid" ) + @Serializer( DeflateStringSerializer.class ) + private String repositoryId; + + @Column( name = "repositoryname" ) + @Serializer( DeflateStringSerializer.class ) + private String repositoryName; //@ManyToOne(cascade = { CascadeType.ALL }, fetch = FetchType.EAGER) //@JoinColumn(name = "repository_id") @@ -64,7 +68,8 @@ public class Namespace { this.id = new KeyBuilder().withNamespace( id ).withRepositoryId( repository.getId() ).build(); this.name = id; - this.repository = repository; + this.repositoryId = repository.getId(); + this.repositoryName = repository.getName(); } public String getId() @@ -89,12 +94,13 @@ public class Namespace public Repository getRepository() { - return repository; + return new Repository(this.repositoryId); } public void setRepository( Repository repository ) { - this.repository = repository; + this.repositoryId = repository.getId(); + this.repositoryName = repository.getName(); } @Override @@ -115,7 +121,7 @@ public class Namespace { return false; } - if ( !repository.getId().equals( namespace.repository.getId() ) ) + if ( !repositoryId.equals( namespace.repositoryId ) ) { return false; } @@ -127,7 +133,7 @@ public class Namespace public int hashCode() { int result = id.hashCode(); - result = 31 * result + repository.getId().hashCode(); + result = 31 * result + repositoryId.hashCode(); return result; } @@ -137,7 +143,7 @@ public class Namespace final StringBuilder sb = new StringBuilder( "Namespace{" ); sb.append( "id='" ).append( id ).append( '\'' ); sb.append( ", name='" ).append( name ).append( '\'' ); - sb.append( ", repository='" ).append( repository ).append( '\'' ); + sb.append( ", repository='" ).append( repositoryId ).append( '\'' ); sb.append( '}' ); return sb.toString(); } diff --git a/archiva-modules/plugins/pom.xml b/archiva-modules/plugins/pom.xml index 94fdc42ff..a380e3547 100644 --- a/archiva-modules/plugins/pom.xml +++ b/archiva-modules/plugins/pom.xml @@ -39,6 +39,6 @@ stage-repository-merge generic-metadata-support metadata-store-jcr - metadata-store-cassandra + -- 2.39.5