Browse Source

Changing checksum handling in metadata

pull/51/head
Martin Stockhammer 4 years ago
parent
commit
ecec848cff

+ 6
- 0
archiva-modules/metadata/metadata-model/pom.xml View File

@@ -32,10 +32,16 @@
</properties>

<dependencies>
<dependency>
<groupId>org.apache.archiva</groupId>
<artifactId>archiva-checksum</artifactId>
</dependency>
<dependency>
<groupId>org.apache.geronimo.specs</groupId>
<artifactId>geronimo-jpa_2.0_spec</artifactId>
</dependency>
</dependencies>



</project>

+ 109
- 113
archiva-modules/metadata/metadata-model/src/main/java/org/apache/archiva/metadata/model/ArtifactMetadata.java View File

@@ -19,6 +19,7 @@ package org.apache.archiva.metadata.model;
* under the License.
*/

import org.apache.archiva.checksum.ChecksumAlgorithm;
import sun.reflect.generics.repository.MethodRepository;

import javax.xml.bind.annotation.XmlRootElement;
@@ -28,6 +29,10 @@ import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.temporal.TemporalAccessor;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;

/**
* Metadata stored in the content repository for a particular artifact. Information that is shared between different
@@ -37,10 +42,9 @@ import java.util.Date;
* For more information, see the
* <a href="{@docRoot}/../metadata-content-model.html" target="_top">Metadata Content Model</a>.
*/
@XmlRootElement ( name = "artifactMetadata" )
@XmlRootElement(name = "artifactMetadata")
public class ArtifactMetadata
extends FacetedMetadata
{
extends FacetedMetadata {


/**
@@ -57,21 +61,21 @@ public class ArtifactMetadata
/**
* The namespace of the project within the repository.
*
* @see org.apache.archiva.metadata.model.ProjectMetadata#namespace
* @see ProjectMetadata#getNamespace()
*/
private String namespace;

/**
* The identifier of the project within the repository and namespace.
*
* @see org.apache.archiva.metadata.model.ProjectMetadata#id
* @see ProjectMetadata#getId()
*/
private String project;

/**
* The version of the project. This may be more generalised than @{link #version}.
*
* @see org.apache.archiva.metadata.model.ProjectVersionMetadata#id
* @see ProjectVersionMetadata#getId()
*/
private String projectVersion;

@@ -93,193 +97,176 @@ public class ArtifactMetadata
private long size;

/**
* The MD5 checksum of the artifact, if calculated.
* The list of checksums.
*/
private String md5;
private Map<ChecksumAlgorithm, String> checksums = new HashMap<>();

/**
* The SHA-1 checksum of the artifact, if calculated.
*/
private String sha1;
private String toStringValue = "";
private int lastHash = 0;

/**
* When the artifact was found in the repository storage and added to the metadata content repository.
*/
private ZonedDateTime whenGathered;

public String getId()
{
public String getId() {
return id;
}

public void setId( String id )
{
public void setId(String id) {
this.id = id;
}

public long getSize()
{
public long getSize() {
return size;
}

public void setSize( long size )
{
public void setSize(long size) {
this.size = size;
}

public String getVersion()
{
public String getVersion() {
return version;
}

public void setVersion( String version )
{
public void setVersion(String version) {
this.version = version;
}

public String getProjectVersion()
{
public String getProjectVersion() {
return projectVersion;
}

public void setProjectVersion( String projectVersion )
{
public void setProjectVersion(String projectVersion) {
this.projectVersion = projectVersion;
}

public void setFileLastModified( long fileLastModified )
{
public void setFileLastModified(long fileLastModified) {
this.fileLastModified = ZonedDateTime.ofInstant(Instant.ofEpochMilli(fileLastModified), ModelInfo.STORAGE_TZ);
}

public void setWhenGathered( ZonedDateTime whenGathered )
{
public void setWhenGathered(ZonedDateTime whenGathered) {
this.whenGathered = whenGathered.withZoneSameInstant(ModelInfo.STORAGE_TZ);
}

public void setMd5( String md5 )
{
this.md5 = md5;
public void setMd5(String md5) {
this.checksums.put(ChecksumAlgorithm.MD5, md5);
}

public void setSha1( String sha1 )
{
this.sha1 = sha1;
public void setSha1(String sha1) {
this.checksums.put(ChecksumAlgorithm.SHA1, sha1);
}

public ZonedDateTime getWhenGathered()
{
public ZonedDateTime getWhenGathered() {
return whenGathered;
}

public String getMd5()
{
return md5;
public String getChecksum(ChecksumAlgorithm checksumAlgorithm) {
return checksums.get(checksumAlgorithm);
}

public void setChecksum(ChecksumAlgorithm algorithm, String checksumValue) {
this.checksums.put(algorithm, checksumValue);
}

public Set<ChecksumAlgorithm> getChecksumTypes() {
return checksums.keySet();
}

public String getSha1()
{
return sha1;
public Map<ChecksumAlgorithm,String> getChecksums() {
return this.checksums;
}

public ZonedDateTime getFileLastModified()
{
public void setChecksums(Map<ChecksumAlgorithm,String> checksums) {
this.checksums = checksums;
}

public String getMd5() {
return checksums.get(ChecksumAlgorithm.MD5);
}

public String getSha1() {
return checksums.get(ChecksumAlgorithm.SHA1);
}

public ZonedDateTime getFileLastModified() {

return fileLastModified;
}

public String getNamespace()
{
public String getNamespace() {
return namespace;
}

public void setNamespace( String namespace )
{
public void setNamespace(String namespace) {
this.namespace = namespace;
}

public void setProject( String project )
{
public void setProject(String project) {
this.project = project;
}

public String getProject()
{
public String getProject() {
return project;
}

public String getRepositoryId()
{
public String getRepositoryId() {
return repositoryId;
}

public void setRepositoryId( String repositoryId )
{
public void setRepositoryId(String repositoryId) {
this.repositoryId = repositoryId;
}

@Override
public boolean equals( Object o )
{
if ( this == o )
{
public boolean equals(Object o) {
if (this == o) {
return true;
}
if ( o == null || getClass() != o.getClass() )
{
if (o == null || getClass() != o.getClass()) {
return false;
}

ArtifactMetadata that = (ArtifactMetadata) o;

if ( size != that.size )
{
if (size != that.size) {
return false;
}
// Time equality by instant that means the point in time must match, but not the time zone
if ( fileLastModified != null
? !fileLastModified.toInstant().equals( that.fileLastModified.toInstant() )
: that.fileLastModified != null )
{
if (fileLastModified != null
? !fileLastModified.toInstant().equals(that.fileLastModified.toInstant())
: that.fileLastModified != null) {
return false;
}
if ( !id.equals( that.id ) )
{
if (!id.equals(that.id)) {
return false;
}
if ( md5 != null ? !md5.equals( that.md5 ) : that.md5 != null )
{
return false;
for ( Map.Entry<ChecksumAlgorithm, String> entry : this.checksums.entrySet()) {
String thatChecksum = that.checksums.get(entry.getKey());
if (entry.getValue()!=null ? !entry.getValue().equals(thatChecksum) : thatChecksum!=null) {
return false;
}
}
if ( namespace != null ? !namespace.equals( that.namespace ) : that.namespace != null )
{
if (namespace != null ? !namespace.equals(that.namespace) : that.namespace != null) {
return false;
}
if ( project != null ? !project.equals( that.project ) : that.project != null )
{
if (project != null ? !project.equals(that.project) : that.project != null) {
return false;
}
if ( projectVersion != null ? !projectVersion.equals( that.projectVersion ) : that.projectVersion != null )
{
if (projectVersion != null ? !projectVersion.equals(that.projectVersion) : that.projectVersion != null) {
return false;
}
/**
* We cannot compare in different repositories, if this is in here
if ( !repositoryId.equals( that.repositoryId ) )
{
return false;
}
if ( !repositoryId.equals( that.repositoryId ) )
{
return false;
}
**/
if ( sha1 != null ? !sha1.equals( that.sha1 ) : that.sha1 != null )
{
return false;
}
if ( version != null ? !version.equals( that.version ) : that.version != null )
{
if (version != null ? !version.equals(that.version) : that.version != null) {
return false;
}
if ( whenGathered != null ? !whenGathered.toInstant().equals( that.whenGathered.toInstant() ) : that.whenGathered != null )
{
if (whenGathered != null ? !whenGathered.toInstant().equals(that.whenGathered.toInstant()) : that.whenGathered != null) {
return false;
}

@@ -287,28 +274,37 @@ public class ArtifactMetadata
}

@Override
public int hashCode()
{
public int hashCode() {
int result = id != null ? id.hashCode() : 0;
result = 31 * result + ( repositoryId != null ? repositoryId.hashCode() : 0 );
result = 31 * result + ( namespace != null ? namespace.hashCode() : 0 );
result = 31 * result + ( project != null ? project.hashCode() : 0 );
result = 31 * result + ( projectVersion != null ? projectVersion.hashCode() : 0 );
result = 31 * result + ( version != null ? version.hashCode() : 0 );
result = 31 * result + ( fileLastModified != null ? fileLastModified.hashCode() : 0 );
result = 31 * result + (int) ( size ^ ( size >>> 32 ) );
result = 31 * result + ( md5 != null ? md5.hashCode() : 0 );
result = 31 * result + ( sha1 != null ? sha1.hashCode() : 0 );
result = 31 * result + ( whenGathered != null ? whenGathered.hashCode() : 0 );
result = 31 * result + (repositoryId != null ? repositoryId.hashCode() : 0);
result = 31 * result + (namespace != null ? namespace.hashCode() : 0);
result = 31 * result + (project != null ? project.hashCode() : 0);
result = 31 * result + (projectVersion != null ? projectVersion.hashCode() : 0);
result = 31 * result + (version != null ? version.hashCode() : 0);
result = 31 * result + (fileLastModified != null ? fileLastModified.hashCode() : 0);
result = 31 * result + (int) (size ^ (size >>> 32));
for (String checksum : checksums.values()) {
result = 31 * result + (checksum != null ? checksum.hashCode() : 0);
}
result = 31 * result + (whenGathered != null ? whenGathered.hashCode() : 0);
return result;
}

/**
* Doing some hashing to avoid the expensive string concatenation.
*/
@Override
public String toString()
{
return "ArtifactMetadata{" + "id='" + id + '\'' + ", size=" + size + ", version='" + version + '\'' +
", fileLastModified=" + fileLastModified + ", whenGathered=" + whenGathered + ", md5='" + md5 + '\'' +
", sha1='" + sha1 + '\'' + ", namespace='" + namespace + '\'' + ", project='" + project + '\'' +
", projectVersion='" + projectVersion + '\'' + ", repositoryId='" + repositoryId + '\'' + '}';
public String toString() {
final int hashCode=hashCode();
if (hashCode!=lastHash) {
toStringValue = "ArtifactMetadata{" + "id='" + id + '\'' + ", size=" + size + ", version='" + version + '\'' +
", fileLastModified=" + fileLastModified + ", whenGathered=" + whenGathered +
", namespace='" + namespace + '\'' + ", project='" + project + '\'' +
", projectVersion='" + projectVersion + '\'' + ", repositoryId='" + repositoryId + '\'' +
", checksums=" + checksums.entrySet().stream().map(e -> e.getKey() + ":" + e.getValue()).collect(Collectors.joining(",")) +
'}';
lastHash=hashCode;
}
return toStringValue;
}
}

+ 21
- 8
archiva-modules/metadata/metadata-repository-api/src/test/java/org/apache/archiva/metadata/repository/AbstractMetadataRepositoryTest.java View File

@@ -80,6 +80,9 @@ public abstract class AbstractMetadataRepositoryTest

protected Logger log = LoggerFactory.getLogger( getClass( ) );

protected int assertMaxTries =10;
protected int assertRetrySleepMs=500;

/*
* Used by tryAssert to allow to throw exceptions in the lambda expression.
*/
@@ -91,7 +94,7 @@ public abstract class AbstractMetadataRepositoryTest

protected void tryAssert( AssertFunction func ) throws Exception
{
tryAssert( func, 20, 500 );
tryAssert( func, assertMaxTries, assertRetrySleepMs );
}


@@ -960,15 +963,25 @@ public abstract class AbstractMetadataRepositoryTest
final ArtifactMetadata artifact1 = createArtifact( );
artifact1.setWhenGathered(ZonedDateTime.now().minusDays(1));
getRepository( ).updateArtifact( session, TEST_REPO_ID, TEST_NAMESPACE, TEST_PROJECT, TEST_PROJECT_VERSION, artifact1 );

ZonedDateTime gatheredNow = ZonedDateTime.now();
final ArtifactMetadata artifact2 = createArtifact( );
artifact2.setId(artifact2.getId()+"-2");
String artifact2Id = artifact2.getId() + "-2";
artifact2.setId(artifact2Id);
artifact2.setVersion(TEST_PROJECT_VERSION+"-2");
artifact2.setWhenGathered(ZonedDateTime.now());
artifact2.setWhenGathered(gatheredNow);
getRepository( ).updateArtifact( session, TEST_REPO_ID, TEST_NAMESPACE, TEST_PROJECT, TEST_PROJECT_VERSION, artifact2 );
final ArtifactMetadata artifact3 = createArtifact();
artifact3.setId(artifact3.getId()+"-3");
String artifact3Id = artifact3.getId() + "-3";
artifact3.setId(artifact3Id);
artifact3.setVersion(TEST_PROJECT_VERSION+"-3");
artifact3.setWhenGathered(ZonedDateTime.now().plusDays(1));
artifact3.setWhenGathered(gatheredNow.minusSeconds(5));

final ArtifactMetadata artifact4 = createArtifact();
artifact4.setId(artifact4.getId()+"-4");
artifact4.setVersion(TEST_PROJECT_VERSION+"-4");
artifact4.setWhenGathered(gatheredNow.plusDays(1));

getRepository( ).updateArtifact( session, TEST_REPO_ID, TEST_NAMESPACE, TEST_PROJECT, TEST_PROJECT_VERSION, artifact3 );
session.save( );

@@ -980,10 +993,10 @@ public abstract class AbstractMetadataRepositoryTest
assertNotNull(stream);

List<ArtifactMetadata> artifacts = stream.collect(Collectors.toList());
assertEquals(1, artifacts.size());

assertEquals(2, artifacts.size());
assertEquals(artifact3Id, artifacts.get(0).getId());
assertEquals(artifact2Id, artifacts.get(1).getId());

assertEquals( Collections.singletonList( artifact2 ), artifacts );
} );
}
}

+ 2
- 0
archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraArchivaManager.java View File

@@ -56,4 +56,6 @@ public interface CassandraArchivaManager

String getDependencyFamilyName();

String getChecksumFamilyName();

}

+ 142
- 40
archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepository.java View File

@@ -37,6 +37,7 @@ import me.prettyprint.hector.api.mutation.MutationResult;
import me.prettyprint.hector.api.mutation.Mutator;
import me.prettyprint.hector.api.query.QueryResult;
import me.prettyprint.hector.api.query.RangeSlicesQuery;
import org.apache.archiva.checksum.ChecksumAlgorithm;
import org.apache.archiva.configuration.ArchivaConfiguration;
import org.apache.archiva.metadata.QueryParameter;
import org.apache.archiva.metadata.model.ArtifactMetadata;
@@ -76,9 +77,11 @@ import java.time.ZonedDateTime;
import java.util.*;
import java.util.function.BiFunction;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;

import static org.apache.archiva.metadata.model.ModelInfo.STORAGE_TZ;
import static org.apache.archiva.metadata.repository.cassandra.CassandraUtils.*;
import static org.apache.archiva.metadata.repository.cassandra.model.ColumnNames.*;

@@ -90,6 +93,7 @@ public class CassandraMetadataRepository
extends AbstractMetadataRepository implements MetadataRepository
{

private static final String ARTIFACT_METADATA_MODEL_KEY = "artifactMetadataModel.key";
private Logger logger = LoggerFactory.getLogger( getClass() );

private ArchivaConfiguration configuration;
@@ -110,6 +114,8 @@ public class CassandraMetadataRepository

private final ColumnFamilyTemplate<String, String> dependencyTemplate;

private final ColumnFamilyTemplate<String, String> checksumTemplate;

private final Keyspace keyspace;

private final StringSerializer ss = StringSerializer.get();
@@ -163,6 +169,12 @@ public class CassandraMetadataRepository
//
StringSerializer.get(), //
StringSerializer.get() );

this.checksumTemplate = new ThriftColumnFamilyTemplate<>( cassandraArchivaManager.getKeyspace(), //
cassandraArchivaManager.getChecksumFamilyName(),
//
StringSerializer.get(), //
StringSerializer.get() );
}


@@ -1036,6 +1048,77 @@ public class CassandraMetadataRepository
return projectVersionMetadata;
}

protected void recordChecksums( String repositoryId, String artifactMetadataKey, Map<String, String> checksums)
{
if ( checksums == null || checksums.isEmpty() )
{
return;
}
Mutator<String> checksumMutator = this.checksumTemplate.createMutator();
for ( Map.Entry<String, String> entry : checksums.entrySet())
{
// we don't care about the key as the real used one with the projectVersionMetadata
String keyChecksums = UUID.randomUUID().toString();
String cfChecksums = cassandraArchivaManager.getChecksumFamilyName();

addInsertion( checksumMutator, keyChecksums, cfChecksums, ARTIFACT_METADATA_MODEL_KEY,
artifactMetadataKey );
addInsertion( checksumMutator, keyChecksums, cfChecksums, CHECKSUM_ALG.toString(), entry.getKey());
addInsertion( checksumMutator, keyChecksums, cfChecksums, CHECKSUM_VALUE.toString(),
entry.getValue() );
addInsertion(checksumMutator, keyChecksums, cfChecksums, REPOSITORY_NAME.toString(), repositoryId);

}
checksumMutator.execute();
}

protected void removeChecksums( String artifactMetadataKey )
{

QueryResult<OrderedRows<String, String, String>> result =
HFactory.createRangeSlicesQuery( cassandraArchivaManager.getKeyspace(), ss, ss, ss ) //
.setColumnFamily( cassandraArchivaManager.getChecksumFamilyName() ) //
.setColumnNames( CHECKSUM_ALG.toString() ) //
.setRowCount( Integer.MAX_VALUE ) //
.addEqualsExpression(ARTIFACT_METADATA_MODEL_KEY, artifactMetadataKey ) //
.execute();

if ( result.get().getCount() < 1 )
{
return;
}

for ( Row<String, String, String> row : result.get() )
{
this.checksumTemplate.deleteRow( row.getKey() );
}

}

protected Map<String, String> getChecksums( String artifactMetadataKey )
{
Map<String, String> checksums = new HashMap<>();

QueryResult<OrderedRows<String, String, String>> result =
HFactory.createRangeSlicesQuery( cassandraArchivaManager.getKeyspace(), ss, ss, ss ) //
.setColumnFamily( cassandraArchivaManager.getChecksumFamilyName() ) //
.setColumnNames( ARTIFACT_METADATA_MODEL_KEY, REPOSITORY_NAME.toString(),
CHECKSUM_ALG.toString(), CHECKSUM_VALUE.toString() ) //
.setRowCount( Integer.MAX_VALUE ) //
.addEqualsExpression(ARTIFACT_METADATA_MODEL_KEY, artifactMetadataKey) //
.execute();
for ( Row<String, String, String> row : result.get() )
{
ColumnFamilyResult<String, String> columnFamilyResult =
this.checksumTemplate.queryColumns( row.getKey() );

checksums.put(columnFamilyResult.getString(CHECKSUM_ALG.toString()),
columnFamilyResult.getString(CHECKSUM_VALUE.toString()));
}

return checksums;
}

protected void recordMailingList( String projectVersionMetadataKey, List<MailingList> mailingLists )
{
if ( mailingLists == null || mailingLists.isEmpty() )
@@ -1297,6 +1380,18 @@ public class CassandraMetadataRepository
return dependencies;
}

private Map<String, String> mapChecksums(Map<ChecksumAlgorithm,String> checksums) {
return checksums.entrySet().stream().collect(Collectors.toMap(
e -> e.getKey().name(), e -> e.getValue()
));
}

private Map<ChecksumAlgorithm, String> mapChecksumsReverse(Map<String,String> checksums) {
return checksums.entrySet().stream().collect(Collectors.toMap(
e -> ChecksumAlgorithm.valueOf(e.getKey()), e -> e.getValue()
));
}

@Override
public void updateArtifact( RepositorySession session, String repositoryId, String namespaceId, String projectId, String projectVersion,
ArtifactMetadata artifactMeta )
@@ -1328,9 +1423,9 @@ public class CassandraMetadataRepository
updater.setLong( FILE_LAST_MODIFIED.toString(), artifactMeta.getFileLastModified().toInstant().toEpochMilli());
updater.setLong( WHEN_GATHERED.toString(), artifactMeta.getWhenGathered().toInstant().toEpochMilli() );
updater.setLong( SIZE.toString(), artifactMeta.getSize() );
addUpdateStringValue( updater, MD5.toString(), artifactMeta.getMd5() );
addUpdateStringValue( updater, SHA1.toString(), artifactMeta.getSha1() );
addUpdateStringValue( updater, VERSION.toString(), artifactMeta.getVersion() );
removeChecksums(key);
recordChecksums(repositoryId, key, mapChecksums(artifactMeta.getChecksums()));
this.artifactMetadataTemplate.update( updater );
}
else
@@ -1346,10 +1441,9 @@ public class CassandraMetadataRepository
.addInsertion( key, cf, column( VERSION.toString(), artifactMeta.getVersion() ) ) //
.addInsertion( key, cf, column( FILE_LAST_MODIFIED.toString(), artifactMeta.getFileLastModified().toInstant().toEpochMilli() ) ) //
.addInsertion( key, cf, column( SIZE.toString(), artifactMeta.getSize() ) ) //
.addInsertion( key, cf, column( MD5.toString(), artifactMeta.getMd5() ) ) //
.addInsertion( key, cf, column( SHA1.toString(), artifactMeta.getSha1() ) ) //
.addInsertion( key, cf, column( WHEN_GATHERED.toString(), artifactMeta.getWhenGathered().toInstant().toEpochMilli() ) )//
.execute();
recordChecksums(repositoryId, key, mapChecksums(artifactMeta.getChecksums()));
}

key = new ProjectVersionMetadataModel.KeyBuilder() //
@@ -1397,6 +1491,7 @@ public class CassandraMetadataRepository
artifactMetadataModel.setFileLastModified( artifactMeta.getFileLastModified() == null
? ZonedDateTime.now().toInstant().toEpochMilli()
: artifactMeta.getFileLastModified().toInstant().toEpochMilli() );
artifactMetadataModel.setChecksums(mapChecksums(artifactMeta.getChecksums()));

// now facets
updateFacets( artifactMeta, artifactMetadataModel );
@@ -1419,7 +1514,7 @@ public class CassandraMetadataRepository
.addEqualsExpression( PROJECT_VERSION.toString(), projectVersion ) //
.execute();

final Set<String> versions = new HashSet<String>();
final Set<String> versions = new HashSet<>();

for ( Row<String, String, String> row : result.get() )
{
@@ -1806,15 +1901,16 @@ public class CassandraMetadataRepository
QueryResult<OrderedRows<String, String, Long>> result = query.execute();

List<ArtifactMetadata> artifactMetadatas = new ArrayList<>( result.get().getCount() );

for ( Row<String, String, Long> row : result.get() )
{
ColumnSlice<String, Long> columnSlice = row.getColumnSlice();
String repositoryName = getAsStringValue( columnSlice, REPOSITORY_NAME.toString() );
if ( StringUtils.equals( repositoryName, repositoryId ) )
{

artifactMetadatas.add( mapArtifactMetadataLongColumnSlice( columnSlice ) );
Iterator<Row<String, String, Long>> keyIter = result.get().iterator();
if (keyIter.hasNext()) {
String key = keyIter.next().getKey();
for (Row<String, String, Long> row : result.get()) {
ColumnSlice<String, Long> columnSlice = row.getColumnSlice();
String repositoryName = getAsStringValue(columnSlice, REPOSITORY_NAME.toString());
if (StringUtils.equals(repositoryName, repositoryId)) {

artifactMetadatas.add(mapArtifactMetadataLongColumnSlice(key, columnSlice));
}
}
}

@@ -1843,7 +1939,7 @@ public class CassandraMetadataRepository
}


protected ArtifactMetadata mapArtifactMetadataLongColumnSlice( ColumnSlice<String, Long> columnSlice )
protected ArtifactMetadata mapArtifactMetadataLongColumnSlice( String key, ColumnSlice<String, Long> columnSlice )
{
ArtifactMetadata artifactMetadata = new ArtifactMetadata();
artifactMetadata.setNamespace( getAsStringValue( columnSlice, NAMESPACE_ID.toString() ) );
@@ -1859,12 +1955,13 @@ public class CassandraMetadataRepository
Long whenGathered = getLongValue( columnSlice, WHEN_GATHERED.toString() );
if ( whenGathered != null )
{
artifactMetadata.setWhenGathered(ZonedDateTime.ofInstant(Instant.ofEpochMilli(whenGathered), ZoneId.of("GMT")));
artifactMetadata.setWhenGathered(ZonedDateTime.ofInstant(Instant.ofEpochMilli(whenGathered), STORAGE_TZ));
}
artifactMetadata.setChecksums(mapChecksumsReverse(getChecksums(key)));
return artifactMetadata;
}

protected ArtifactMetadata mapArtifactMetadataStringColumnSlice( ColumnSlice<String, String> columnSlice )
protected ArtifactMetadata mapArtifactMetadataStringColumnSlice( String key, ColumnSlice<String, String> columnSlice )
{
ArtifactMetadata artifactMetadata = new ArtifactMetadata();
artifactMetadata.setNamespace( getStringValue( columnSlice, NAMESPACE_ID.toString() ) );
@@ -1880,8 +1977,9 @@ public class CassandraMetadataRepository
Long whenGathered = getAsLongValue( columnSlice, WHEN_GATHERED.toString() );
if ( whenGathered != null )
{
artifactMetadata.setWhenGathered(ZonedDateTime.ofInstant(Instant.ofEpochMilli(whenGathered), ZoneId.of("GMT")));
artifactMetadata.setWhenGathered(ZonedDateTime.ofInstant(Instant.ofEpochMilli(whenGathered), STORAGE_TZ));
}
artifactMetadata.setChecksums(mapChecksumsReverse(getChecksums(key)));
return artifactMetadata;
}

@@ -1895,37 +1993,37 @@ public class CassandraMetadataRepository

RangeSlicesQuery<String, String, String> query = HFactory //
.createRangeSlicesQuery( keyspace, ss, ss, ss ) //
.setColumnFamily( cassandraArchivaManager.getArtifactMetadataFamilyName() ) //
.setColumnNames( ArtifactMetadataModel.COLUMNS ); //
.setColumnFamily( cassandraArchivaManager.getChecksumFamilyName()) //
.setColumnNames(ARTIFACT_METADATA_MODEL_KEY); //

query = query.addEqualsExpression( SHA1.toString(), checksum ).addEqualsExpression( REPOSITORY_NAME.toString(), repositoryId );
query = query.addEqualsExpression( CHECKSUM_VALUE.toString(), checksum )
.addEqualsExpression( REPOSITORY_NAME.toString(), repositoryId );

QueryResult<OrderedRows<String, String, String>> result = query.execute();

List<String> artifactKeys = new ArrayList<>();
for ( Row<String, String, String> row : result.get() )
{
ColumnSlice<String, String> columnSlice = row.getColumnSlice();

artifactMetadataMap.put( row.getKey(), mapArtifactMetadataStringColumnSlice( columnSlice ) );
artifactKeys.add(columnSlice.getColumnByName(ARTIFACT_METADATA_MODEL_KEY).getValue());

}

query = HFactory //
.createRangeSlicesQuery( keyspace, ss, ss, ss ) //
.setColumnFamily( cassandraArchivaManager.getArtifactMetadataFamilyName() ) //
.setColumnNames( NAMESPACE_ID.toString(), SIZE.toString(), ID.toString(), FILE_LAST_MODIFIED.toString(), MD5.toString(), PROJECT.toString(), PROJECT_VERSION.toString(),
REPOSITORY_NAME.toString(), VERSION.toString(), WHEN_GATHERED.toString(), SHA1.toString() ); //

query = query.addEqualsExpression( MD5.toString(), checksum ).addEqualsExpression( REPOSITORY_NAME.toString(), repositoryId );

result = query.execute();

for ( Row<String, String, String> row : result.get() )
{
ColumnSlice<String, String> columnSlice = row.getColumnSlice();
for (String key : artifactKeys) {
query = HFactory //
.createRangeSlicesQuery(keyspace, ss, ss, ss) //
.setColumnFamily(cassandraArchivaManager.getArtifactMetadataFamilyName()) //
.setColumnNames(NAMESPACE_ID.toString(), SIZE.toString(), ID.toString(), FILE_LAST_MODIFIED.toString(), MD5.toString(), PROJECT.toString(), PROJECT_VERSION.toString(),
REPOSITORY_NAME.toString(), VERSION.toString(), WHEN_GATHERED.toString(), SHA1.toString())
.setKeys(key, key);
result = query.execute();

artifactMetadataMap.put( row.getKey(), mapArtifactMetadataStringColumnSlice( columnSlice ) );
for (Row<String, String, String> row : result.get()) {
ColumnSlice<String, String> columnSlice = row.getColumnSlice();

artifactMetadataMap.put(row.getKey(), mapArtifactMetadataStringColumnSlice(key, columnSlice));
}
}

return new ArrayList(artifactMetadataMap.values());
@@ -1993,7 +2091,8 @@ public class CassandraMetadataRepository

for ( Row<String, String, String> artifactMetadataRow : artifactMetadataResult.get() )
{
artifactMetadatas.add( mapArtifactMetadataStringColumnSlice( artifactMetadataRow.getColumnSlice() ) );
String artifactKey = artifactMetadataRow.getKey();
artifactMetadatas.add( mapArtifactMetadataStringColumnSlice( artifactKey, artifactMetadataRow.getColumnSlice() ) );
}
}

@@ -2120,13 +2219,15 @@ public class CassandraMetadataRepository

QueryResult<OrderedRows<String, String, String>> result = query.execute();



List<ArtifactMetadata> artifactMetadatas = new ArrayList<>( result.get().getCount() );

for ( Row<String, String, String> row : result.get() )
{
String key = row.getKey();
ColumnSlice<String, String> columnSlice = row.getColumnSlice();

artifactMetadatas.add( mapArtifactMetadataStringColumnSlice( columnSlice ) );
artifactMetadatas.add( mapArtifactMetadataStringColumnSlice( key, columnSlice ) );

}

@@ -2240,7 +2341,8 @@ public class CassandraMetadataRepository

for ( Row<String, String, String> row : result.get() )
{
artifactMetadatas.add( mapArtifactMetadataStringColumnSlice( row.getColumnSlice() ) );
String key = row.getKey();
artifactMetadatas.add( mapArtifactMetadataStringColumnSlice( key, row.getColumnSlice() ) );
}

result = HFactory.createRangeSlicesQuery( keyspace, ss, ss, ss ) //

+ 50
- 0
archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/DefaultCassandraArchivaManager.java View File

@@ -92,6 +92,8 @@ public class DefaultCassandraArchivaManager

private String dependencyFamilyName = "dependency";

private String checksumFamilyName = "checksum";

@Value("${cassandra.host}")
private String cassandraHost;

@@ -388,6 +390,49 @@ public class DefaultCassandraArchivaManager

}

// Checksum table
{
final ColumnFamilyDefinition checksumCf =
HFactory.createColumnFamilyDefinition( keyspace.getKeyspaceName(), //
getChecksumFamilyName(), //
ComparatorType.UTF8TYPE );

BasicColumnDefinition artifactMetatadaModel_key = new BasicColumnDefinition();
artifactMetatadaModel_key.setName( StringSerializer.get().toByteBuffer( "artifactMetadataModel.key" ) );
artifactMetatadaModel_key.setIndexName( "artifactMetadataModel_key" );
artifactMetatadaModel_key.setIndexType( ColumnIndexType.KEYS );
artifactMetatadaModel_key.setValidationClass( ComparatorType.UTF8TYPE.getClassName() );
checksumCf.addColumnDefinition( artifactMetatadaModel_key );


BasicColumnDefinition checksumAlgorithmColumn = new BasicColumnDefinition();
checksumAlgorithmColumn.setName( StringSerializer.get().toByteBuffer( CHECKSUM_ALG.toString() ) );
checksumAlgorithmColumn.setIndexName( CHECKSUM_ALG.toString() );
checksumAlgorithmColumn.setIndexType( ColumnIndexType.KEYS );
checksumAlgorithmColumn.setValidationClass( ComparatorType.UTF8TYPE.getClassName() );
checksumCf.addColumnDefinition( checksumAlgorithmColumn );

BasicColumnDefinition checksumValueColumn = new BasicColumnDefinition();
checksumValueColumn.setName( StringSerializer.get().toByteBuffer( CHECKSUM_VALUE.toString() ) );
checksumValueColumn.setIndexName( CHECKSUM_VALUE.toString() );
checksumValueColumn.setIndexType( ColumnIndexType.KEYS );
checksumValueColumn.setValidationClass( ComparatorType.UTF8TYPE.getClassName() );
checksumCf.addColumnDefinition( checksumValueColumn );

BasicColumnDefinition repositoryNameColumn = new BasicColumnDefinition();
repositoryNameColumn.setName( StringSerializer.get().toByteBuffer( REPOSITORY_NAME.toString() ) );
repositoryNameColumn.setIndexName( REPOSITORY_NAME.toString() );
repositoryNameColumn.setIndexType( ColumnIndexType.KEYS );
repositoryNameColumn.setValidationClass( ComparatorType.UTF8TYPE.getClassName() );
checksumCf.addColumnDefinition( repositoryNameColumn );


cfds.add( checksumCf );

// creating indexes for cql query

}

// mailinglist table
{
final ColumnFamilyDefinition mailingListCf =
@@ -553,4 +598,9 @@ public class DefaultCassandraArchivaManager
{
return dependencyFamilyName;
}

@Override
public String getChecksumFamilyName() {
return checksumFamilyName;
}
}

+ 20
- 0
archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/model/ArtifactMetadataModel.java View File

@@ -23,6 +23,8 @@ import org.apache.archiva.metadata.repository.cassandra.CassandraUtils;

import java.io.Serializable;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;

import static org.apache.archiva.metadata.repository.cassandra.model.ColumnNames.*;

@@ -62,6 +64,8 @@ public class ArtifactMetadataModel

private long whenGathered;

private Map<String, String> checksums = new HashMap<>();

public ArtifactMetadataModel()
{
// no op
@@ -195,6 +199,22 @@ public class ArtifactMetadataModel
this.whenGathered = whenGathered;
}

public void setChecksum(String type, String value) {
this.checksums.put(type, value);
}

public String getChecksum(String type) {
return this.checksums.get(type);
}

public void setChecksums(Map<String,String> checksums) {
this.checksums = checksums;
}

public Map<String,String> getChecksums() {
return this.checksums;
}


@Override
public String toString()

+ 3
- 1
archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/model/ColumnNames.java View File

@@ -45,7 +45,9 @@ public enum ColumnNames
ARTIFACT_ID( "artifactId" ),
DESCRIPTION( "description" ),
URL( "url" ),
WHEN_GATHERED( "whenGathered" );
WHEN_GATHERED( "whenGathered" ),
CHECKSUM_ALG("checksumAlgorithm"),
CHECKSUM_VALUE("checksumValue");

private final String name;


+ 3
- 1
archiva-modules/plugins/metadata-store-cassandra/src/test/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepositoryTest.java View File

@@ -38,7 +38,6 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import java.util.Map;

import static org.assertj.core.api.Assertions.assertThat;

@@ -79,7 +78,10 @@ public class CassandraMetadataRepositoryTest
public void setUp()
throws Exception
{

super.setUp();
assertMaxTries =1;
assertRetrySleepMs=10;

Path directory = Paths.get( "target/test-repositories" );
if ( Files.exists(directory) )

Loading…
Cancel
Save