From 4e5d1183c31651680e8266198f41c4464647b28d Mon Sep 17 00:00:00 2001 From: Carlos Sanchez Date: Sun, 23 Nov 2014 11:34:24 +0100 Subject: [PATCH] [MRM-1390] Implement search methods for generic metadata and properties in Cassandra store Some caveats with Cassandra implementation: We can't search artifacts by any (wildcard) property, so searchArtifacts(text,...) just calls getArtifactsByMetadata The exact parameter is ignored as we can't do non exact searches in Cassandra --- .../CassandraMetadataRepository.java | 141 ++++++++++++++---- .../repository/cassandra/CassandraUtils.java | 8 +- .../cassandra/model/MetadataFacetModel.java | 5 + .../CassandraMetadataRepositoryTest.java | 98 ------------ .../repository/jcr/JcrMetadataRepository.java | 3 + 5 files changed, 127 insertions(+), 128 deletions(-) diff --git a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepository.java b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepository.java index 779c19f1c..268fd157e 100644 --- a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepository.java +++ b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepository.java @@ -77,6 +77,7 @@ import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; @@ -1826,25 +1827,114 @@ public class CassandraMetadataRepository return artifactMetadataMap.values(); } + /** + * Project version and artifact level metadata are stored in the same place, no distinctions in Cassandra + * implementation, just calls {@link #getArtifactsByMetadata(String, String, String)} + */ @Override public List getArtifactsByProjectVersionMetadata( String key, String value, String repositoryId ) throws MetadataRepositoryException { - throw new UnsupportedOperationException( "not yet implemented in Cassandra backend" ); + return getArtifactsByMetadata( key, value, repositoryId ); } @Override public List getArtifactsByMetadata( String key, String value, String repositoryId ) throws MetadataRepositoryException { - throw new UnsupportedOperationException( "not yet implemented in Cassandra backend" ); + RangeSlicesQuery query = + HFactory.createRangeSlicesQuery( keyspace, ss, ss, ss ) // + .setColumnFamily( cassandraArchivaManager.getMetadataFacetFamilyName() ) // + .setColumnNames( MetadataFacetModel.COLUMNS ) // + .addEqualsExpression( VALUE.toString(), value ); + + if ( key != null ) + { + query.addEqualsExpression( KEY.toString(), key ); // + } + if ( repositoryId != null ) + { + query.addEqualsExpression( "repositoryName", repositoryId ); + } + + QueryResult> metadataFacetResult = query.execute(); + if ( metadataFacetResult.get() == null || metadataFacetResult.get().getCount() < 1 ) + { + return Collections.emptyList(); + } + + List artifactMetadatas = new LinkedList(); + + // TODO doing multiple queries, there should be a way to get all the artifactMetadatas for any number of + // projects + for ( Row row : metadataFacetResult.get() ) + { + QueryResult> artifactMetadataResult = + HFactory.createRangeSlicesQuery( keyspace, ss, ss, ss ) // + .setColumnFamily( cassandraArchivaManager.getArtifactMetadataFamilyName() ) // + .setColumnNames( ArtifactMetadataModel.COLUMNS ) // + .setRowCount( Integer.MAX_VALUE ) // + .addEqualsExpression( REPOSITORY_NAME.toString(), + getStringValue( row.getColumnSlice(), REPOSITORY_NAME ) ) // + .addEqualsExpression( NAMESPACE_ID.toString(), getStringValue( row.getColumnSlice(), NAMESPACE_ID ) ) // + .addEqualsExpression( PROJECT.toString(), getStringValue( row.getColumnSlice(), PROJECT_ID ) ) // + .addEqualsExpression( PROJECT_VERSION.toString(), + getStringValue( row.getColumnSlice(), PROJECT_VERSION ) ) // + .execute(); + + if ( artifactMetadataResult.get() == null || artifactMetadataResult.get().getCount() < 1 ) + { + return Collections.emptyList(); + } + + for ( Row artifactMetadataRow : artifactMetadataResult.get() ) + { + artifactMetadatas.add( mapArtifactMetadataStringColumnSlice( artifactMetadataRow.getColumnSlice() ) ); + } + } + + return mapArtifactMetadataToArtifact( metadataFacetResult, artifactMetadatas ); } @Override public List getArtifactsByProperty( String key, String value, String repositoryId ) throws MetadataRepositoryException { - throw new UnsupportedOperationException( "getArtifactsByProperty not yet implemented in Cassandra backend" ); + QueryResult> result = + HFactory.createRangeSlicesQuery( keyspace, ss, ss, ss ) // + .setColumnFamily( cassandraArchivaManager.getProjectVersionMetadataFamilyName() ) // + .setColumnNames( PROJECT_ID.toString(), REPOSITORY_NAME.toString(), NAMESPACE_ID.toString(), + PROJECT_VERSION.toString() ) // + .addEqualsExpression( key, value ) // + .execute(); + + int count = result.get().getCount(); + + if ( count < 1 ) + { + return Collections.emptyList(); + } + + List artifacts = new LinkedList(); + + for ( Row row : result.get() ) + { + // TODO doing multiple queries, there should be a way to get all the artifactMetadatas for any number of + // projects + try + { + artifacts.addAll( getArtifacts( getStringValue( row.getColumnSlice(), REPOSITORY_NAME ), + getStringValue( row.getColumnSlice(), NAMESPACE_ID ), + getStringValue( row.getColumnSlice(), PROJECT_ID ), + getStringValue( row.getColumnSlice(), PROJECT_VERSION ) ) ); + } + catch ( MetadataResolutionException e ) + { + // never raised + throw new IllegalStateException( e ); + } + } + return artifacts; } @Override @@ -2046,29 +2136,12 @@ public class CassandraMetadataRepository for ( Row row : result.get() ) { - ColumnSlice columnSlice = row.getColumnSlice(); - ArtifactMetadata artifactMetadata = new ArtifactMetadata(); - artifactMetadata.setNamespace( getStringValue( columnSlice, NAMESPACE_ID.toString() ) ); - artifactMetadata.setSize( getAsLongValue( columnSlice, SIZE.toString() ) ); - artifactMetadata.setId( getStringValue( columnSlice, ID.toString() ) ); - artifactMetadata.setFileLastModified( getAsLongValue( columnSlice, FILE_LAST_MODIFIED.toString() ) ); - artifactMetadata.setMd5( getStringValue( columnSlice, MD5.toString() ) ); - artifactMetadata.setProject( getStringValue( columnSlice, PROJECT.toString() ) ); - artifactMetadata.setProjectVersion( getStringValue( columnSlice, PROJECT_VERSION.toString() ) ); - artifactMetadata.setRepositoryId( repoId ); - artifactMetadata.setSha1( getStringValue( columnSlice, SHA1.toString() ) ); - artifactMetadata.setVersion( getStringValue( columnSlice, VERSION.toString() ) ); - Long whenGathered = getAsLongValue( columnSlice, WHEN_GATHERED.toString() ); - if ( whenGathered != null ) - { - artifactMetadata.setWhenGathered( new Date( whenGathered ) ); - } - artifactMetadatas.add( artifactMetadata ); + artifactMetadatas.add( mapArtifactMetadataStringColumnSlice( row.getColumnSlice() ) ); } result = HFactory.createRangeSlicesQuery( keyspace, ss, ss, ss ) // .setColumnFamily( cassandraArchivaManager.getMetadataFacetFamilyName() ) // - .setColumnNames( FACET_ID.toString(), NAME.toString(), VALUE.toString(), KEY.toString(), PROJECT_VERSION.toString() ) // + .setColumnNames( MetadataFacetModel.COLUMNS ) // .setRowCount( Integer.MAX_VALUE ) // .addEqualsExpression( REPOSITORY_NAME.toString(), repoId ) // .addEqualsExpression( NAMESPACE_ID.toString(), namespace ) // @@ -2076,6 +2149,13 @@ public class CassandraMetadataRepository .addEqualsExpression( PROJECT_VERSION.toString(), projectVersion ) // .execute(); + return mapArtifactMetadataToArtifact(result, artifactMetadatas); + } + + /** + * Attach metadata to each of the ArtifactMetadata objects + */ + private List mapArtifactMetadataToArtifact(QueryResult> result, List artifactMetadatas) { if ( result.get() == null || result.get().getCount() < 1 ) { return artifactMetadatas; @@ -2147,8 +2227,6 @@ public class CassandraMetadataRepository } } } - - } return artifactMetadatas; @@ -2198,17 +2276,28 @@ public class CassandraMetadataRepository return ModelMapperHolder.MODEL_MAPPER; } + /** + * This implementation just calls getArtifactsByMetadata( null, text, repositoryId ). We can't search artifacts by + * any property. + */ @Override public List searchArtifacts( String text, String repositoryId, boolean exact ) throws MetadataRepositoryException { - throw new UnsupportedOperationException( "searchArtifacts not yet implemented in Cassandra backend" ); + return getArtifactsByMetadata( null, text, repositoryId ); } + /** + * The exact parameter is ignored as we can't do non exact searches in Cassandra + */ @Override public List searchArtifacts( String key, String text, String repositoryId, boolean exact ) throws MetadataRepositoryException { - throw new UnsupportedOperationException( "searchArtifacts not yet implemented in Cassandra backend" ); + // TODO optimize + List artifacts = new LinkedList(); + artifacts.addAll( getArtifactsByMetadata( key, text, repositoryId ) ); + artifacts.addAll( getArtifactsByProperty( key, text, repositoryId ) ); + return artifacts; } } diff --git a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraUtils.java b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraUtils.java index 6366fe9f8..f3de42b90 100644 --- a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraUtils.java +++ b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraUtils.java @@ -95,18 +95,18 @@ public class CassandraUtils return hColumn == null ? null : hColumn.getValue(); } - public static Long getLongValue( ColumnSlice columnSlice, String columnName ) + public static Long getLongValue( ColumnSlice columnSlice, String columnName ) { if ( StringUtils.isEmpty( columnName ) ) { return null; } - HColumn hColumn = columnSlice.getColumnByName( columnName ); + HColumn hColumn = (HColumn) columnSlice.getColumnByName( columnName ); return hColumn == null ? null : hColumn.getValue(); } - public static String getAsStringValue( ColumnSlice columnSlice, String columnName ) + public static String getAsStringValue( ColumnSlice columnSlice, String columnName ) { StringSerializer ss = StringSerializer.get(); if ( StringUtils.isEmpty( columnName ) ) @@ -114,7 +114,7 @@ public class CassandraUtils return null; } - HColumn hColumn = columnSlice.getColumnByName( columnName ); + HColumn hColumn = columnSlice.getColumnByName( columnName ); return hColumn == null ? null : ss.fromByteBuffer( hColumn.getValueBytes() ); } diff --git a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/model/MetadataFacetModel.java b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/model/MetadataFacetModel.java index ae494e1ab..a2347bf1b 100644 --- a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/model/MetadataFacetModel.java +++ b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/model/MetadataFacetModel.java @@ -19,6 +19,8 @@ package org.apache.archiva.metadata.repository.cassandra.model; * under the License. */ +import static org.apache.archiva.metadata.repository.cassandra.model.ColumnNames.*; + import org.apache.archiva.metadata.repository.cassandra.CassandraUtils; /** @@ -29,6 +31,9 @@ import org.apache.archiva.metadata.repository.cassandra.CassandraUtils; */ public class MetadataFacetModel { + public static final String[] COLUMNS = new String[] { FACET_ID.toString(), KEY.toString(), VALUE.toString(), + REPOSITORY_NAME.toString(), NAMESPACE_ID.toString(), PROJECT_ID.toString(), PROJECT_VERSION.toString() }; + private String facetId; private String key; diff --git a/archiva-modules/plugins/metadata-store-cassandra/src/test/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepositoryTest.java b/archiva-modules/plugins/metadata-store-cassandra/src/test/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepositoryTest.java index 97b3718f7..b4f0cdb82 100644 --- a/archiva-modules/plugins/metadata-store-cassandra/src/test/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepositoryTest.java +++ b/archiva-modules/plugins/metadata-store-cassandra/src/test/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepositoryTest.java @@ -25,7 +25,6 @@ import org.apache.archiva.metadata.repository.cassandra.model.ProjectVersionMeta import org.apache.commons.io.FileUtils; import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import javax.inject.Inject; @@ -69,103 +68,6 @@ public class CassandraMetadataRepositoryTest clearReposAndNamespace( cassandraArchivaManager ); } - @Override - @Ignore - public void testGetArtifactsByProjectVersionMetadata() - throws Exception - { - // TODO not implemented - } - - @Override - @Ignore - public void testGetArtifactsByProjectVersionMetadataNoRepository() - throws Exception - { - // TODO not implemented - } - - @Override - @Ignore - public void testGetArtifactsByProjectVersionMetadataAllRepositories() - throws Exception - { - // TODO not implemented - } - - @Override - @Ignore - public void testGetArtifactsByMetadataAllRepositories() - throws Exception - { - // TODO not implemented - } - - @Override - @Ignore - public void testGetArtifactsByPropertySingleResult() - throws Exception - { - // TODO not implemented - } - - @Override - @Ignore - public void testSearchArtifactsByKey() - throws Exception - { - // TODO not implemented - } - - @Override - @Ignore - public void testSearchArtifactsByKeyExact() - throws Exception - { - // TODO not implemented - } - - @Override - @Ignore - public void testSearchArtifactsFullText() - throws Exception - { - // TODO not implemented - } - - @Override - @Ignore - public void testSearchArtifactsFullTextExact() - throws Exception - { - // TODO not implemented - } - - @Override - @Ignore - public void testSearchArtifactsByFacetKeyAllRepos() - throws Exception - { - // TODO not implemented - } - - @Override - @Ignore - public void testSearchArtifactsByFacetKey() - throws Exception - { - // TODO not implemented - } - - @Override - @Ignore - public void testSearchArtifactsFullTextByFacet() - throws Exception - { - // TODO not implemented - } - - /** * ensure all dependant tables are cleaned up (mailinglist, license, dependencies) * diff --git a/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/JcrMetadataRepository.java b/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/JcrMetadataRepository.java index d4c79b603..8fecba8a5 100644 --- a/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/JcrMetadataRepository.java +++ b/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/JcrMetadataRepository.java @@ -1403,6 +1403,9 @@ public class JcrMetadataRepository } + /** + * Exact is ignored as we can't do exact search in any property, we need a key + */ @Override public List searchArtifacts( String text, String repositoryId, boolean exact ) throws MetadataRepositoryException -- 2.39.5