diff options
author | Martin Stockhammer <martin_s@apache.org> | 2019-08-24 17:09:57 +0200 |
---|---|---|
committer | Martin Stockhammer <martin_s@apache.org> | 2019-08-24 17:09:57 +0200 |
commit | ecec848cff05ca987e517c3fb8eef9a1e5f01121 (patch) | |
tree | 40a2abe17aa77f4d7e644c5fbbadd44b5bb09066 | |
parent | 31e8442faddc796db59426a5749ecb64ad76acf5 (diff) | |
download | archiva-ecec848cff05ca987e517c3fb8eef9a1e5f01121.tar.gz archiva-ecec848cff05ca987e517c3fb8eef9a1e5f01121.zip |
Changing checksum handling in metadata
9 files changed, 356 insertions, 163 deletions
diff --git a/archiva-modules/metadata/metadata-model/pom.xml b/archiva-modules/metadata/metadata-model/pom.xml index 4faf4781f..4d81ad961 100644 --- a/archiva-modules/metadata/metadata-model/pom.xml +++ b/archiva-modules/metadata/metadata-model/pom.xml @@ -33,9 +33,15 @@ <dependencies> <dependency> + <groupId>org.apache.archiva</groupId> + <artifactId>archiva-checksum</artifactId> + </dependency> + <dependency> <groupId>org.apache.geronimo.specs</groupId> <artifactId>geronimo-jpa_2.0_spec</artifactId> </dependency> </dependencies> + + </project> diff --git a/archiva-modules/metadata/metadata-model/src/main/java/org/apache/archiva/metadata/model/ArtifactMetadata.java b/archiva-modules/metadata/metadata-model/src/main/java/org/apache/archiva/metadata/model/ArtifactMetadata.java index 8bb12d3dd..fd44febfd 100644 --- a/archiva-modules/metadata/metadata-model/src/main/java/org/apache/archiva/metadata/model/ArtifactMetadata.java +++ b/archiva-modules/metadata/metadata-model/src/main/java/org/apache/archiva/metadata/model/ArtifactMetadata.java @@ -19,6 +19,7 @@ package org.apache.archiva.metadata.model; * under the License. */ +import org.apache.archiva.checksum.ChecksumAlgorithm; import sun.reflect.generics.repository.MethodRepository; import javax.xml.bind.annotation.XmlRootElement; @@ -28,6 +29,10 @@ import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.temporal.TemporalAccessor; import java.util.Date; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; /** * Metadata stored in the content repository for a particular artifact. Information that is shared between different @@ -37,10 +42,9 @@ import java.util.Date; * For more information, see the * <a href="{@docRoot}/../metadata-content-model.html" target="_top">Metadata Content Model</a>. */ -@XmlRootElement ( name = "artifactMetadata" ) +@XmlRootElement(name = "artifactMetadata") public class ArtifactMetadata - extends FacetedMetadata -{ + extends FacetedMetadata { /** @@ -57,21 +61,21 @@ public class ArtifactMetadata /** * The namespace of the project within the repository. * - * @see org.apache.archiva.metadata.model.ProjectMetadata#namespace + * @see ProjectMetadata#getNamespace() */ private String namespace; /** * The identifier of the project within the repository and namespace. * - * @see org.apache.archiva.metadata.model.ProjectMetadata#id + * @see ProjectMetadata#getId() */ private String project; /** * The version of the project. This may be more generalised than @{link #version}. * - * @see org.apache.archiva.metadata.model.ProjectVersionMetadata#id + * @see ProjectVersionMetadata#getId() */ private String projectVersion; @@ -93,193 +97,176 @@ public class ArtifactMetadata private long size; /** - * The MD5 checksum of the artifact, if calculated. + * The list of checksums. */ - private String md5; + private Map<ChecksumAlgorithm, String> checksums = new HashMap<>(); - /** - * The SHA-1 checksum of the artifact, if calculated. - */ - private String sha1; + private String toStringValue = ""; + private int lastHash = 0; /** * When the artifact was found in the repository storage and added to the metadata content repository. */ private ZonedDateTime whenGathered; - public String getId() - { + public String getId() { return id; } - public void setId( String id ) - { + public void setId(String id) { this.id = id; } - public long getSize() - { + public long getSize() { return size; } - public void setSize( long size ) - { + public void setSize(long size) { this.size = size; } - public String getVersion() - { + public String getVersion() { return version; } - public void setVersion( String version ) - { + public void setVersion(String version) { this.version = version; } - public String getProjectVersion() - { + public String getProjectVersion() { return projectVersion; } - public void setProjectVersion( String projectVersion ) - { + public void setProjectVersion(String projectVersion) { this.projectVersion = projectVersion; } - public void setFileLastModified( long fileLastModified ) - { + public void setFileLastModified(long fileLastModified) { this.fileLastModified = ZonedDateTime.ofInstant(Instant.ofEpochMilli(fileLastModified), ModelInfo.STORAGE_TZ); } - public void setWhenGathered( ZonedDateTime whenGathered ) - { + public void setWhenGathered(ZonedDateTime whenGathered) { this.whenGathered = whenGathered.withZoneSameInstant(ModelInfo.STORAGE_TZ); } - public void setMd5( String md5 ) - { - this.md5 = md5; + public void setMd5(String md5) { + this.checksums.put(ChecksumAlgorithm.MD5, md5); } - public void setSha1( String sha1 ) - { - this.sha1 = sha1; + public void setSha1(String sha1) { + this.checksums.put(ChecksumAlgorithm.SHA1, sha1); } - public ZonedDateTime getWhenGathered() - { + public ZonedDateTime getWhenGathered() { return whenGathered; } - public String getMd5() - { - return md5; + public String getChecksum(ChecksumAlgorithm checksumAlgorithm) { + return checksums.get(checksumAlgorithm); + } + + public void setChecksum(ChecksumAlgorithm algorithm, String checksumValue) { + this.checksums.put(algorithm, checksumValue); + } + + public Set<ChecksumAlgorithm> getChecksumTypes() { + return checksums.keySet(); } - public String getSha1() - { - return sha1; + public Map<ChecksumAlgorithm,String> getChecksums() { + return this.checksums; } - public ZonedDateTime getFileLastModified() - { + public void setChecksums(Map<ChecksumAlgorithm,String> checksums) { + this.checksums = checksums; + } + + public String getMd5() { + return checksums.get(ChecksumAlgorithm.MD5); + } + + public String getSha1() { + return checksums.get(ChecksumAlgorithm.SHA1); + } + + public ZonedDateTime getFileLastModified() { return fileLastModified; } - public String getNamespace() - { + public String getNamespace() { return namespace; } - public void setNamespace( String namespace ) - { + public void setNamespace(String namespace) { this.namespace = namespace; } - public void setProject( String project ) - { + public void setProject(String project) { this.project = project; } - public String getProject() - { + public String getProject() { return project; } - public String getRepositoryId() - { + public String getRepositoryId() { return repositoryId; } - public void setRepositoryId( String repositoryId ) - { + public void setRepositoryId(String repositoryId) { this.repositoryId = repositoryId; } @Override - public boolean equals( Object o ) - { - if ( this == o ) - { + public boolean equals(Object o) { + if (this == o) { return true; } - if ( o == null || getClass() != o.getClass() ) - { + if (o == null || getClass() != o.getClass()) { return false; } ArtifactMetadata that = (ArtifactMetadata) o; - if ( size != that.size ) - { + if (size != that.size) { return false; } // Time equality by instant that means the point in time must match, but not the time zone - if ( fileLastModified != null - ? !fileLastModified.toInstant().equals( that.fileLastModified.toInstant() ) - : that.fileLastModified != null ) - { + if (fileLastModified != null + ? !fileLastModified.toInstant().equals(that.fileLastModified.toInstant()) + : that.fileLastModified != null) { return false; } - if ( !id.equals( that.id ) ) - { + if (!id.equals(that.id)) { return false; } - if ( md5 != null ? !md5.equals( that.md5 ) : that.md5 != null ) - { - return false; + for ( Map.Entry<ChecksumAlgorithm, String> entry : this.checksums.entrySet()) { + String thatChecksum = that.checksums.get(entry.getKey()); + if (entry.getValue()!=null ? !entry.getValue().equals(thatChecksum) : thatChecksum!=null) { + return false; + } } - if ( namespace != null ? !namespace.equals( that.namespace ) : that.namespace != null ) - { + if (namespace != null ? !namespace.equals(that.namespace) : that.namespace != null) { return false; } - if ( project != null ? !project.equals( that.project ) : that.project != null ) - { + if (project != null ? !project.equals(that.project) : that.project != null) { return false; } - if ( projectVersion != null ? !projectVersion.equals( that.projectVersion ) : that.projectVersion != null ) - { + if (projectVersion != null ? !projectVersion.equals(that.projectVersion) : that.projectVersion != null) { return false; } /** * We cannot compare in different repositories, if this is in here - if ( !repositoryId.equals( that.repositoryId ) ) - { - return false; - } + if ( !repositoryId.equals( that.repositoryId ) ) + { + return false; + } **/ - if ( sha1 != null ? !sha1.equals( that.sha1 ) : that.sha1 != null ) - { - return false; - } - if ( version != null ? !version.equals( that.version ) : that.version != null ) - { + if (version != null ? !version.equals(that.version) : that.version != null) { return false; } - if ( whenGathered != null ? !whenGathered.toInstant().equals( that.whenGathered.toInstant() ) : that.whenGathered != null ) - { + if (whenGathered != null ? !whenGathered.toInstant().equals(that.whenGathered.toInstant()) : that.whenGathered != null) { return false; } @@ -287,28 +274,37 @@ public class ArtifactMetadata } @Override - public int hashCode() - { + public int hashCode() { int result = id != null ? id.hashCode() : 0; - result = 31 * result + ( repositoryId != null ? repositoryId.hashCode() : 0 ); - result = 31 * result + ( namespace != null ? namespace.hashCode() : 0 ); - result = 31 * result + ( project != null ? project.hashCode() : 0 ); - result = 31 * result + ( projectVersion != null ? projectVersion.hashCode() : 0 ); - result = 31 * result + ( version != null ? version.hashCode() : 0 ); - result = 31 * result + ( fileLastModified != null ? fileLastModified.hashCode() : 0 ); - result = 31 * result + (int) ( size ^ ( size >>> 32 ) ); - result = 31 * result + ( md5 != null ? md5.hashCode() : 0 ); - result = 31 * result + ( sha1 != null ? sha1.hashCode() : 0 ); - result = 31 * result + ( whenGathered != null ? whenGathered.hashCode() : 0 ); + result = 31 * result + (repositoryId != null ? repositoryId.hashCode() : 0); + result = 31 * result + (namespace != null ? namespace.hashCode() : 0); + result = 31 * result + (project != null ? project.hashCode() : 0); + result = 31 * result + (projectVersion != null ? projectVersion.hashCode() : 0); + result = 31 * result + (version != null ? version.hashCode() : 0); + result = 31 * result + (fileLastModified != null ? fileLastModified.hashCode() : 0); + result = 31 * result + (int) (size ^ (size >>> 32)); + for (String checksum : checksums.values()) { + result = 31 * result + (checksum != null ? checksum.hashCode() : 0); + } + result = 31 * result + (whenGathered != null ? whenGathered.hashCode() : 0); return result; } + /** + * Doing some hashing to avoid the expensive string concatenation. + */ @Override - public String toString() - { - return "ArtifactMetadata{" + "id='" + id + '\'' + ", size=" + size + ", version='" + version + '\'' + - ", fileLastModified=" + fileLastModified + ", whenGathered=" + whenGathered + ", md5='" + md5 + '\'' + - ", sha1='" + sha1 + '\'' + ", namespace='" + namespace + '\'' + ", project='" + project + '\'' + - ", projectVersion='" + projectVersion + '\'' + ", repositoryId='" + repositoryId + '\'' + '}'; + public String toString() { + final int hashCode=hashCode(); + if (hashCode!=lastHash) { + toStringValue = "ArtifactMetadata{" + "id='" + id + '\'' + ", size=" + size + ", version='" + version + '\'' + + ", fileLastModified=" + fileLastModified + ", whenGathered=" + whenGathered + + ", namespace='" + namespace + '\'' + ", project='" + project + '\'' + + ", projectVersion='" + projectVersion + '\'' + ", repositoryId='" + repositoryId + '\'' + + ", checksums=" + checksums.entrySet().stream().map(e -> e.getKey() + ":" + e.getValue()).collect(Collectors.joining(",")) + + '}'; + lastHash=hashCode; + } + return toStringValue; } } diff --git a/archiva-modules/metadata/metadata-repository-api/src/test/java/org/apache/archiva/metadata/repository/AbstractMetadataRepositoryTest.java b/archiva-modules/metadata/metadata-repository-api/src/test/java/org/apache/archiva/metadata/repository/AbstractMetadataRepositoryTest.java index a747b7f28..c777097bb 100644 --- a/archiva-modules/metadata/metadata-repository-api/src/test/java/org/apache/archiva/metadata/repository/AbstractMetadataRepositoryTest.java +++ b/archiva-modules/metadata/metadata-repository-api/src/test/java/org/apache/archiva/metadata/repository/AbstractMetadataRepositoryTest.java @@ -80,6 +80,9 @@ public abstract class AbstractMetadataRepositoryTest protected Logger log = LoggerFactory.getLogger( getClass( ) ); + protected int assertMaxTries =10; + protected int assertRetrySleepMs=500; + /* * Used by tryAssert to allow to throw exceptions in the lambda expression. */ @@ -91,7 +94,7 @@ public abstract class AbstractMetadataRepositoryTest protected void tryAssert( AssertFunction func ) throws Exception { - tryAssert( func, 20, 500 ); + tryAssert( func, assertMaxTries, assertRetrySleepMs ); } @@ -960,15 +963,25 @@ public abstract class AbstractMetadataRepositoryTest final ArtifactMetadata artifact1 = createArtifact( ); artifact1.setWhenGathered(ZonedDateTime.now().minusDays(1)); getRepository( ).updateArtifact( session, TEST_REPO_ID, TEST_NAMESPACE, TEST_PROJECT, TEST_PROJECT_VERSION, artifact1 ); + + ZonedDateTime gatheredNow = ZonedDateTime.now(); final ArtifactMetadata artifact2 = createArtifact( ); - artifact2.setId(artifact2.getId()+"-2"); + String artifact2Id = artifact2.getId() + "-2"; + artifact2.setId(artifact2Id); artifact2.setVersion(TEST_PROJECT_VERSION+"-2"); - artifact2.setWhenGathered(ZonedDateTime.now()); + artifact2.setWhenGathered(gatheredNow); getRepository( ).updateArtifact( session, TEST_REPO_ID, TEST_NAMESPACE, TEST_PROJECT, TEST_PROJECT_VERSION, artifact2 ); final ArtifactMetadata artifact3 = createArtifact(); - artifact3.setId(artifact3.getId()+"-3"); + String artifact3Id = artifact3.getId() + "-3"; + artifact3.setId(artifact3Id); artifact3.setVersion(TEST_PROJECT_VERSION+"-3"); - artifact3.setWhenGathered(ZonedDateTime.now().plusDays(1)); + artifact3.setWhenGathered(gatheredNow.minusSeconds(5)); + + final ArtifactMetadata artifact4 = createArtifact(); + artifact4.setId(artifact4.getId()+"-4"); + artifact4.setVersion(TEST_PROJECT_VERSION+"-4"); + artifact4.setWhenGathered(gatheredNow.plusDays(1)); + getRepository( ).updateArtifact( session, TEST_REPO_ID, TEST_NAMESPACE, TEST_PROJECT, TEST_PROJECT_VERSION, artifact3 ); session.save( ); @@ -980,10 +993,10 @@ public abstract class AbstractMetadataRepositoryTest assertNotNull(stream); List<ArtifactMetadata> artifacts = stream.collect(Collectors.toList()); - assertEquals(1, artifacts.size()); - + assertEquals(2, artifacts.size()); + assertEquals(artifact3Id, artifacts.get(0).getId()); + assertEquals(artifact2Id, artifacts.get(1).getId()); - assertEquals( Collections.singletonList( artifact2 ), artifacts ); } ); } } diff --git a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraArchivaManager.java b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraArchivaManager.java index 45af48360..41ac95a24 100644 --- a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraArchivaManager.java +++ b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraArchivaManager.java @@ -56,4 +56,6 @@ public interface CassandraArchivaManager String getDependencyFamilyName(); + String getChecksumFamilyName(); + } diff --git a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepository.java b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepository.java index 97a57533d..768a94bef 100644 --- a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepository.java +++ b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepository.java @@ -37,6 +37,7 @@ import me.prettyprint.hector.api.mutation.MutationResult; import me.prettyprint.hector.api.mutation.Mutator; import me.prettyprint.hector.api.query.QueryResult; import me.prettyprint.hector.api.query.RangeSlicesQuery; +import org.apache.archiva.checksum.ChecksumAlgorithm; import org.apache.archiva.configuration.ArchivaConfiguration; import org.apache.archiva.metadata.QueryParameter; import org.apache.archiva.metadata.model.ArtifactMetadata; @@ -76,9 +77,11 @@ import java.time.ZonedDateTime; import java.util.*; import java.util.function.BiFunction; import java.util.function.Consumer; +import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.stream.StreamSupport; +import static org.apache.archiva.metadata.model.ModelInfo.STORAGE_TZ; import static org.apache.archiva.metadata.repository.cassandra.CassandraUtils.*; import static org.apache.archiva.metadata.repository.cassandra.model.ColumnNames.*; @@ -90,6 +93,7 @@ public class CassandraMetadataRepository extends AbstractMetadataRepository implements MetadataRepository { + private static final String ARTIFACT_METADATA_MODEL_KEY = "artifactMetadataModel.key"; private Logger logger = LoggerFactory.getLogger( getClass() ); private ArchivaConfiguration configuration; @@ -110,6 +114,8 @@ public class CassandraMetadataRepository private final ColumnFamilyTemplate<String, String> dependencyTemplate; + private final ColumnFamilyTemplate<String, String> checksumTemplate; + private final Keyspace keyspace; private final StringSerializer ss = StringSerializer.get(); @@ -163,6 +169,12 @@ public class CassandraMetadataRepository // StringSerializer.get(), // StringSerializer.get() ); + + this.checksumTemplate = new ThriftColumnFamilyTemplate<>( cassandraArchivaManager.getKeyspace(), // + cassandraArchivaManager.getChecksumFamilyName(), + // + StringSerializer.get(), // + StringSerializer.get() ); } @@ -1036,6 +1048,77 @@ public class CassandraMetadataRepository return projectVersionMetadata; } + protected void recordChecksums( String repositoryId, String artifactMetadataKey, Map<String, String> checksums) + { + if ( checksums == null || checksums.isEmpty() ) + { + return; + } + Mutator<String> checksumMutator = this.checksumTemplate.createMutator(); + for ( Map.Entry<String, String> entry : checksums.entrySet()) + { + // we don't care about the key as the real used one with the projectVersionMetadata + String keyChecksums = UUID.randomUUID().toString(); + String cfChecksums = cassandraArchivaManager.getChecksumFamilyName(); + + addInsertion( checksumMutator, keyChecksums, cfChecksums, ARTIFACT_METADATA_MODEL_KEY, + artifactMetadataKey ); + addInsertion( checksumMutator, keyChecksums, cfChecksums, CHECKSUM_ALG.toString(), entry.getKey()); + addInsertion( checksumMutator, keyChecksums, cfChecksums, CHECKSUM_VALUE.toString(), + entry.getValue() ); + addInsertion(checksumMutator, keyChecksums, cfChecksums, REPOSITORY_NAME.toString(), repositoryId); + + } + checksumMutator.execute(); + } + + protected void removeChecksums( String artifactMetadataKey ) + { + + QueryResult<OrderedRows<String, String, String>> result = + HFactory.createRangeSlicesQuery( cassandraArchivaManager.getKeyspace(), ss, ss, ss ) // + .setColumnFamily( cassandraArchivaManager.getChecksumFamilyName() ) // + .setColumnNames( CHECKSUM_ALG.toString() ) // + .setRowCount( Integer.MAX_VALUE ) // + .addEqualsExpression(ARTIFACT_METADATA_MODEL_KEY, artifactMetadataKey ) // + .execute(); + + if ( result.get().getCount() < 1 ) + { + return; + } + + for ( Row<String, String, String> row : result.get() ) + { + this.checksumTemplate.deleteRow( row.getKey() ); + } + + } + + protected Map<String, String> getChecksums( String artifactMetadataKey ) + { + Map<String, String> checksums = new HashMap<>(); + + QueryResult<OrderedRows<String, String, String>> result = + HFactory.createRangeSlicesQuery( cassandraArchivaManager.getKeyspace(), ss, ss, ss ) // + .setColumnFamily( cassandraArchivaManager.getChecksumFamilyName() ) // + .setColumnNames( ARTIFACT_METADATA_MODEL_KEY, REPOSITORY_NAME.toString(), + CHECKSUM_ALG.toString(), CHECKSUM_VALUE.toString() ) // + .setRowCount( Integer.MAX_VALUE ) // + .addEqualsExpression(ARTIFACT_METADATA_MODEL_KEY, artifactMetadataKey) // + .execute(); + for ( Row<String, String, String> row : result.get() ) + { + ColumnFamilyResult<String, String> columnFamilyResult = + this.checksumTemplate.queryColumns( row.getKey() ); + + checksums.put(columnFamilyResult.getString(CHECKSUM_ALG.toString()), + columnFamilyResult.getString(CHECKSUM_VALUE.toString())); + } + + return checksums; + } + protected void recordMailingList( String projectVersionMetadataKey, List<MailingList> mailingLists ) { if ( mailingLists == null || mailingLists.isEmpty() ) @@ -1297,6 +1380,18 @@ public class CassandraMetadataRepository return dependencies; } + private Map<String, String> mapChecksums(Map<ChecksumAlgorithm,String> checksums) { + return checksums.entrySet().stream().collect(Collectors.toMap( + e -> e.getKey().name(), e -> e.getValue() + )); + } + + private Map<ChecksumAlgorithm, String> mapChecksumsReverse(Map<String,String> checksums) { + return checksums.entrySet().stream().collect(Collectors.toMap( + e -> ChecksumAlgorithm.valueOf(e.getKey()), e -> e.getValue() + )); + } + @Override public void updateArtifact( RepositorySession session, String repositoryId, String namespaceId, String projectId, String projectVersion, ArtifactMetadata artifactMeta ) @@ -1328,9 +1423,9 @@ public class CassandraMetadataRepository updater.setLong( FILE_LAST_MODIFIED.toString(), artifactMeta.getFileLastModified().toInstant().toEpochMilli()); updater.setLong( WHEN_GATHERED.toString(), artifactMeta.getWhenGathered().toInstant().toEpochMilli() ); updater.setLong( SIZE.toString(), artifactMeta.getSize() ); - addUpdateStringValue( updater, MD5.toString(), artifactMeta.getMd5() ); - addUpdateStringValue( updater, SHA1.toString(), artifactMeta.getSha1() ); addUpdateStringValue( updater, VERSION.toString(), artifactMeta.getVersion() ); + removeChecksums(key); + recordChecksums(repositoryId, key, mapChecksums(artifactMeta.getChecksums())); this.artifactMetadataTemplate.update( updater ); } else @@ -1346,10 +1441,9 @@ public class CassandraMetadataRepository .addInsertion( key, cf, column( VERSION.toString(), artifactMeta.getVersion() ) ) // .addInsertion( key, cf, column( FILE_LAST_MODIFIED.toString(), artifactMeta.getFileLastModified().toInstant().toEpochMilli() ) ) // .addInsertion( key, cf, column( SIZE.toString(), artifactMeta.getSize() ) ) // - .addInsertion( key, cf, column( MD5.toString(), artifactMeta.getMd5() ) ) // - .addInsertion( key, cf, column( SHA1.toString(), artifactMeta.getSha1() ) ) // .addInsertion( key, cf, column( WHEN_GATHERED.toString(), artifactMeta.getWhenGathered().toInstant().toEpochMilli() ) )// .execute(); + recordChecksums(repositoryId, key, mapChecksums(artifactMeta.getChecksums())); } key = new ProjectVersionMetadataModel.KeyBuilder() // @@ -1397,6 +1491,7 @@ public class CassandraMetadataRepository artifactMetadataModel.setFileLastModified( artifactMeta.getFileLastModified() == null ? ZonedDateTime.now().toInstant().toEpochMilli() : artifactMeta.getFileLastModified().toInstant().toEpochMilli() ); + artifactMetadataModel.setChecksums(mapChecksums(artifactMeta.getChecksums())); // now facets updateFacets( artifactMeta, artifactMetadataModel ); @@ -1419,7 +1514,7 @@ public class CassandraMetadataRepository .addEqualsExpression( PROJECT_VERSION.toString(), projectVersion ) // .execute(); - final Set<String> versions = new HashSet<String>(); + final Set<String> versions = new HashSet<>(); for ( Row<String, String, String> row : result.get() ) { @@ -1806,15 +1901,16 @@ public class CassandraMetadataRepository QueryResult<OrderedRows<String, String, Long>> result = query.execute(); List<ArtifactMetadata> artifactMetadatas = new ArrayList<>( result.get().getCount() ); - - for ( Row<String, String, Long> row : result.get() ) - { - ColumnSlice<String, Long> columnSlice = row.getColumnSlice(); - String repositoryName = getAsStringValue( columnSlice, REPOSITORY_NAME.toString() ); - if ( StringUtils.equals( repositoryName, repositoryId ) ) - { - - artifactMetadatas.add( mapArtifactMetadataLongColumnSlice( columnSlice ) ); + Iterator<Row<String, String, Long>> keyIter = result.get().iterator(); + if (keyIter.hasNext()) { + String key = keyIter.next().getKey(); + for (Row<String, String, Long> row : result.get()) { + ColumnSlice<String, Long> columnSlice = row.getColumnSlice(); + String repositoryName = getAsStringValue(columnSlice, REPOSITORY_NAME.toString()); + if (StringUtils.equals(repositoryName, repositoryId)) { + + artifactMetadatas.add(mapArtifactMetadataLongColumnSlice(key, columnSlice)); + } } } @@ -1843,7 +1939,7 @@ public class CassandraMetadataRepository } - protected ArtifactMetadata mapArtifactMetadataLongColumnSlice( ColumnSlice<String, Long> columnSlice ) + protected ArtifactMetadata mapArtifactMetadataLongColumnSlice( String key, ColumnSlice<String, Long> columnSlice ) { ArtifactMetadata artifactMetadata = new ArtifactMetadata(); artifactMetadata.setNamespace( getAsStringValue( columnSlice, NAMESPACE_ID.toString() ) ); @@ -1859,12 +1955,13 @@ public class CassandraMetadataRepository Long whenGathered = getLongValue( columnSlice, WHEN_GATHERED.toString() ); if ( whenGathered != null ) { - artifactMetadata.setWhenGathered(ZonedDateTime.ofInstant(Instant.ofEpochMilli(whenGathered), ZoneId.of("GMT"))); + artifactMetadata.setWhenGathered(ZonedDateTime.ofInstant(Instant.ofEpochMilli(whenGathered), STORAGE_TZ)); } + artifactMetadata.setChecksums(mapChecksumsReverse(getChecksums(key))); return artifactMetadata; } - protected ArtifactMetadata mapArtifactMetadataStringColumnSlice( ColumnSlice<String, String> columnSlice ) + protected ArtifactMetadata mapArtifactMetadataStringColumnSlice( String key, ColumnSlice<String, String> columnSlice ) { ArtifactMetadata artifactMetadata = new ArtifactMetadata(); artifactMetadata.setNamespace( getStringValue( columnSlice, NAMESPACE_ID.toString() ) ); @@ -1880,8 +1977,9 @@ public class CassandraMetadataRepository Long whenGathered = getAsLongValue( columnSlice, WHEN_GATHERED.toString() ); if ( whenGathered != null ) { - artifactMetadata.setWhenGathered(ZonedDateTime.ofInstant(Instant.ofEpochMilli(whenGathered), ZoneId.of("GMT"))); + artifactMetadata.setWhenGathered(ZonedDateTime.ofInstant(Instant.ofEpochMilli(whenGathered), STORAGE_TZ)); } + artifactMetadata.setChecksums(mapChecksumsReverse(getChecksums(key))); return artifactMetadata; } @@ -1895,37 +1993,37 @@ public class CassandraMetadataRepository RangeSlicesQuery<String, String, String> query = HFactory // .createRangeSlicesQuery( keyspace, ss, ss, ss ) // - .setColumnFamily( cassandraArchivaManager.getArtifactMetadataFamilyName() ) // - .setColumnNames( ArtifactMetadataModel.COLUMNS ); // + .setColumnFamily( cassandraArchivaManager.getChecksumFamilyName()) // + .setColumnNames(ARTIFACT_METADATA_MODEL_KEY); // - query = query.addEqualsExpression( SHA1.toString(), checksum ).addEqualsExpression( REPOSITORY_NAME.toString(), repositoryId ); + query = query.addEqualsExpression( CHECKSUM_VALUE.toString(), checksum ) + .addEqualsExpression( REPOSITORY_NAME.toString(), repositoryId ); QueryResult<OrderedRows<String, String, String>> result = query.execute(); + List<String> artifactKeys = new ArrayList<>(); for ( Row<String, String, String> row : result.get() ) { ColumnSlice<String, String> columnSlice = row.getColumnSlice(); - artifactMetadataMap.put( row.getKey(), mapArtifactMetadataStringColumnSlice( columnSlice ) ); + artifactKeys.add(columnSlice.getColumnByName(ARTIFACT_METADATA_MODEL_KEY).getValue()); } - query = HFactory // - .createRangeSlicesQuery( keyspace, ss, ss, ss ) // - .setColumnFamily( cassandraArchivaManager.getArtifactMetadataFamilyName() ) // - .setColumnNames( NAMESPACE_ID.toString(), SIZE.toString(), ID.toString(), FILE_LAST_MODIFIED.toString(), MD5.toString(), PROJECT.toString(), PROJECT_VERSION.toString(), - REPOSITORY_NAME.toString(), VERSION.toString(), WHEN_GATHERED.toString(), SHA1.toString() ); // - - query = query.addEqualsExpression( MD5.toString(), checksum ).addEqualsExpression( REPOSITORY_NAME.toString(), repositoryId ); - - result = query.execute(); - - for ( Row<String, String, String> row : result.get() ) - { - ColumnSlice<String, String> columnSlice = row.getColumnSlice(); + for (String key : artifactKeys) { + query = HFactory // + .createRangeSlicesQuery(keyspace, ss, ss, ss) // + .setColumnFamily(cassandraArchivaManager.getArtifactMetadataFamilyName()) // + .setColumnNames(NAMESPACE_ID.toString(), SIZE.toString(), ID.toString(), FILE_LAST_MODIFIED.toString(), MD5.toString(), PROJECT.toString(), PROJECT_VERSION.toString(), + REPOSITORY_NAME.toString(), VERSION.toString(), WHEN_GATHERED.toString(), SHA1.toString()) + .setKeys(key, key); + result = query.execute(); - artifactMetadataMap.put( row.getKey(), mapArtifactMetadataStringColumnSlice( columnSlice ) ); + for (Row<String, String, String> row : result.get()) { + ColumnSlice<String, String> columnSlice = row.getColumnSlice(); + artifactMetadataMap.put(row.getKey(), mapArtifactMetadataStringColumnSlice(key, columnSlice)); + } } return new ArrayList(artifactMetadataMap.values()); @@ -1993,7 +2091,8 @@ public class CassandraMetadataRepository for ( Row<String, String, String> artifactMetadataRow : artifactMetadataResult.get() ) { - artifactMetadatas.add( mapArtifactMetadataStringColumnSlice( artifactMetadataRow.getColumnSlice() ) ); + String artifactKey = artifactMetadataRow.getKey(); + artifactMetadatas.add( mapArtifactMetadataStringColumnSlice( artifactKey, artifactMetadataRow.getColumnSlice() ) ); } } @@ -2120,13 +2219,15 @@ public class CassandraMetadataRepository QueryResult<OrderedRows<String, String, String>> result = query.execute(); + + List<ArtifactMetadata> artifactMetadatas = new ArrayList<>( result.get().getCount() ); for ( Row<String, String, String> row : result.get() ) { + String key = row.getKey(); ColumnSlice<String, String> columnSlice = row.getColumnSlice(); - - artifactMetadatas.add( mapArtifactMetadataStringColumnSlice( columnSlice ) ); + artifactMetadatas.add( mapArtifactMetadataStringColumnSlice( key, columnSlice ) ); } @@ -2240,7 +2341,8 @@ public class CassandraMetadataRepository for ( Row<String, String, String> row : result.get() ) { - artifactMetadatas.add( mapArtifactMetadataStringColumnSlice( row.getColumnSlice() ) ); + String key = row.getKey(); + artifactMetadatas.add( mapArtifactMetadataStringColumnSlice( key, row.getColumnSlice() ) ); } result = HFactory.createRangeSlicesQuery( keyspace, ss, ss, ss ) // diff --git a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/DefaultCassandraArchivaManager.java b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/DefaultCassandraArchivaManager.java index 76381b680..2d8ec7d0e 100644 --- a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/DefaultCassandraArchivaManager.java +++ b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/DefaultCassandraArchivaManager.java @@ -92,6 +92,8 @@ public class DefaultCassandraArchivaManager private String dependencyFamilyName = "dependency"; + private String checksumFamilyName = "checksum"; + @Value("${cassandra.host}") private String cassandraHost; @@ -388,6 +390,49 @@ public class DefaultCassandraArchivaManager } + // Checksum table + { + final ColumnFamilyDefinition checksumCf = + HFactory.createColumnFamilyDefinition( keyspace.getKeyspaceName(), // + getChecksumFamilyName(), // + ComparatorType.UTF8TYPE ); + + BasicColumnDefinition artifactMetatadaModel_key = new BasicColumnDefinition(); + artifactMetatadaModel_key.setName( StringSerializer.get().toByteBuffer( "artifactMetadataModel.key" ) ); + artifactMetatadaModel_key.setIndexName( "artifactMetadataModel_key" ); + artifactMetatadaModel_key.setIndexType( ColumnIndexType.KEYS ); + artifactMetatadaModel_key.setValidationClass( ComparatorType.UTF8TYPE.getClassName() ); + checksumCf.addColumnDefinition( artifactMetatadaModel_key ); + + + BasicColumnDefinition checksumAlgorithmColumn = new BasicColumnDefinition(); + checksumAlgorithmColumn.setName( StringSerializer.get().toByteBuffer( CHECKSUM_ALG.toString() ) ); + checksumAlgorithmColumn.setIndexName( CHECKSUM_ALG.toString() ); + checksumAlgorithmColumn.setIndexType( ColumnIndexType.KEYS ); + checksumAlgorithmColumn.setValidationClass( ComparatorType.UTF8TYPE.getClassName() ); + checksumCf.addColumnDefinition( checksumAlgorithmColumn ); + + BasicColumnDefinition checksumValueColumn = new BasicColumnDefinition(); + checksumValueColumn.setName( StringSerializer.get().toByteBuffer( CHECKSUM_VALUE.toString() ) ); + checksumValueColumn.setIndexName( CHECKSUM_VALUE.toString() ); + checksumValueColumn.setIndexType( ColumnIndexType.KEYS ); + checksumValueColumn.setValidationClass( ComparatorType.UTF8TYPE.getClassName() ); + checksumCf.addColumnDefinition( checksumValueColumn ); + + BasicColumnDefinition repositoryNameColumn = new BasicColumnDefinition(); + repositoryNameColumn.setName( StringSerializer.get().toByteBuffer( REPOSITORY_NAME.toString() ) ); + repositoryNameColumn.setIndexName( REPOSITORY_NAME.toString() ); + repositoryNameColumn.setIndexType( ColumnIndexType.KEYS ); + repositoryNameColumn.setValidationClass( ComparatorType.UTF8TYPE.getClassName() ); + checksumCf.addColumnDefinition( repositoryNameColumn ); + + + cfds.add( checksumCf ); + + // creating indexes for cql query + + } + // mailinglist table { final ColumnFamilyDefinition mailingListCf = @@ -553,4 +598,9 @@ public class DefaultCassandraArchivaManager { return dependencyFamilyName; } + + @Override + public String getChecksumFamilyName() { + return checksumFamilyName; + } } diff --git a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/model/ArtifactMetadataModel.java b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/model/ArtifactMetadataModel.java index db0499786..64ba9a37e 100644 --- a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/model/ArtifactMetadataModel.java +++ b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/model/ArtifactMetadataModel.java @@ -23,6 +23,8 @@ import org.apache.archiva.metadata.repository.cassandra.CassandraUtils; import java.io.Serializable; import java.util.Date; +import java.util.HashMap; +import java.util.Map; import static org.apache.archiva.metadata.repository.cassandra.model.ColumnNames.*; @@ -62,6 +64,8 @@ public class ArtifactMetadataModel private long whenGathered; + private Map<String, String> checksums = new HashMap<>(); + public ArtifactMetadataModel() { // no op @@ -195,6 +199,22 @@ public class ArtifactMetadataModel this.whenGathered = whenGathered; } + public void setChecksum(String type, String value) { + this.checksums.put(type, value); + } + + public String getChecksum(String type) { + return this.checksums.get(type); + } + + public void setChecksums(Map<String,String> checksums) { + this.checksums = checksums; + } + + public Map<String,String> getChecksums() { + return this.checksums; + } + @Override public String toString() diff --git a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/model/ColumnNames.java b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/model/ColumnNames.java index fa8193560..8ea2f389f 100644 --- a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/model/ColumnNames.java +++ b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/model/ColumnNames.java @@ -45,7 +45,9 @@ public enum ColumnNames ARTIFACT_ID( "artifactId" ), DESCRIPTION( "description" ), URL( "url" ), - WHEN_GATHERED( "whenGathered" ); + WHEN_GATHERED( "whenGathered" ), + CHECKSUM_ALG("checksumAlgorithm"), + CHECKSUM_VALUE("checksumValue"); private final String name; diff --git a/archiva-modules/plugins/metadata-store-cassandra/src/test/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepositoryTest.java b/archiva-modules/plugins/metadata-store-cassandra/src/test/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepositoryTest.java index b395e3657..b095f0533 100644 --- a/archiva-modules/plugins/metadata-store-cassandra/src/test/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepositoryTest.java +++ b/archiva-modules/plugins/metadata-store-cassandra/src/test/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepositoryTest.java @@ -38,7 +38,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.List; -import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; @@ -79,7 +78,10 @@ public class CassandraMetadataRepositoryTest public void setUp() throws Exception { + super.setUp(); + assertMaxTries =1; + assertRetrySleepMs=10; Path directory = Paths.get( "target/test-repositories" ); if ( Files.exists(directory) ) |