From 31e8442faddc796db59426a5749ecb64ad76acf5 Mon Sep 17 00:00:00 2001 From: Martin Stockhammer Date: Wed, 21 Aug 2019 22:44:40 +0200 Subject: Fixing TZ for time storage. Adding stream implementations. --- .../repository/AbstractMetadataRepository.java | 46 +++++++- .../metadata/repository/MetadataRepository.java | 116 +++++++++++++++++---- .../repository/AbstractMetadataRepositoryTest.java | 72 ++++++++++++- 3 files changed, 207 insertions(+), 27 deletions(-) (limited to 'archiva-modules/metadata/metadata-repository-api') diff --git a/archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/AbstractMetadataRepository.java b/archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/AbstractMetadataRepository.java index 7d7258645..c4e27103c 100644 --- a/archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/AbstractMetadataRepository.java +++ b/archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/AbstractMetadataRepository.java @@ -26,11 +26,10 @@ import org.apache.archiva.metadata.model.MetadataFacetFactory; import org.apache.archiva.metadata.model.ProjectMetadata; import org.apache.archiva.metadata.model.ProjectVersionMetadata; import org.apache.archiva.metadata.model.ProjectVersionReference; +import org.apache.commons.collections4.ComparatorUtils; import java.time.ZonedDateTime; -import java.util.Collection; -import java.util.List; -import java.util.Set; +import java.util.*; import java.util.stream.Stream; public abstract class AbstractMetadataRepository @@ -119,7 +118,7 @@ public abstract class AbstractMetadataRepository } @Override - public Collection getArtifactsByChecksum( RepositorySession session, String repositoryId, String checksum ) + public List getArtifactsByChecksum(RepositorySession session, String repositoryId, String checksum ) throws MetadataRepositoryException { throw new UnsupportedOperationException(); @@ -255,6 +254,45 @@ public abstract class AbstractMetadataRepository throw new UnsupportedOperationException(); } + protected static Comparator getArtifactMetadataComparator(final QueryParameter queryParameter, String defaultAttr) { + List> compList = new ArrayList<>(); + List sortFields = new ArrayList<>(); + if (queryParameter.getSortFields().size() == 0) { + sortFields.add(defaultAttr); + } else { + sortFields = queryParameter.getSortFields(); + } + for (String attribute : sortFields) { + switch (attribute) { + case "id": + compList.add(Comparator.comparing(ArtifactMetadata::getId)); + break; + case "whenGathered": + compList.add(Comparator.comparing(ArtifactMetadata::getWhenGathered)); + break; + case "fileLastModified": + compList.add(Comparator.comparing(ArtifactMetadata::getFileLastModified)); + case "version": + compList.add(Comparator.comparing(ArtifactMetadata::getVersion)); + break; + case "projectVersion": + compList.add(Comparator.comparing(ArtifactMetadata::getProjectVersion)); + break; + case "project": + compList.add(Comparator.comparing(ArtifactMetadata::getProject)); + break; + default: + // + } + } + Comparator comp = ComparatorUtils.chainedComparator(compList); + if (queryParameter.isAscending()) { + return comp; + } else { + return comp.reversed(); + } + } + @Override public Collection getArtifacts( RepositorySession session, String repoId, String namespace, String projectId, String projectVersion ) diff --git a/archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/MetadataRepository.java b/archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/MetadataRepository.java index b0849cd14..e679704c8 100644 --- a/archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/MetadataRepository.java +++ b/archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/MetadataRepository.java @@ -26,6 +26,7 @@ import org.apache.archiva.metadata.model.ProjectMetadata; import org.apache.archiva.metadata.model.ProjectVersionMetadata; import org.apache.archiva.metadata.model.ProjectVersionReference; +import java.time.ZoneId; import java.time.ZonedDateTime; import java.util.Collection; import java.util.List; @@ -87,9 +88,16 @@ import java.util.stream.Stream; * * Facets can be stored on repository, project, version and artifact level. * + * For retrieving artifacts there are methods that return lists and streaming based methods. Some implementations (e.g. JCR) use + * lazy loading for the retrieved objects. So the streaming methods may be faster and use less memory than the list based methods. + * But for some backends there is no difference. + * */ public interface MetadataRepository { + + + /** * Update metadata for a particular project in the metadata repository, or create it, if it does not already exist. * @@ -155,15 +163,46 @@ public interface MetadataRepository List getMetadataFacets( RepositorySession session, String repositoryId, String facetId ) throws MetadataRepositoryException; + + /** + * + * The same as + * @see #getMetadataFacetStream(RepositorySession, String, Class, QueryParameter queryParameter) + * but uses default query parameters. + * + * There is no limitation of the number of result objects returned, but implementations may have a hard upper bound for + * the number of results. + * + * @param session + * @param repositoryId + * @param facetClazz + * @param + * @return + * @throws MetadataRepositoryException + * @since 3.0 + */ Stream getMetadataFacetStream( RepositorySession session, String repositoryId, Class facetClazz) throws MetadataRepositoryException; + /** + * Returns a stream of MetadataFacet elements that match the given facet class. + * Implementations should order the resulting stream by facet name. + * + * + * @param session The repository session + * @param repositoryId The repository id + * @param facetClazz The class of the facet + * @param The facet type + * @return + * @throws MetadataRepositoryException + * @since 3.0 + */ Stream getMetadataFacetStream(RepositorySession session, String repositoryId, Class facetClazz, QueryParameter queryParameter) throws MetadataRepositoryException; /** - * Returns true, if there is facet data stored for the given id on the repository. The facet data itself - * may be empty. It's just checking if there is data stored for the given facet id. + * Returns true, if there is facet data stored for the given facet id on the repository on repository level. The facet data itself + * may be empty. It's just checking if there is an object stored for the given facet id. * * @param session The repository session * @param repositoryId The repository id @@ -200,13 +239,14 @@ public interface MetadataRepository * @param name The name of the facet (name or path) * @param The type of the facet object * @return The facet instance, if it exists. - * @throws MetadataRepositoryException + * @throws MetadataRepositoryException if the data cannot be retrieved from the backend + * @since 3.0 */ T getMetadataFacet(RepositorySession session, String repositoryId, Class clazz, String name) throws MetadataRepositoryException; /** - * Adss a facet to the repository level. + * Adds a facet to the repository level. * * @param session The repository session * @param repositoryId The id of the repository @@ -238,41 +278,79 @@ public interface MetadataRepository void removeMetadataFacet( RepositorySession session, String repositoryId, String facetId, String name ) throws MetadataRepositoryException; + /** - * if startTime or endTime are null they are not used for search - * + * Is the same as {@link #getArtifactsByDateRange(RepositorySession, String, ZonedDateTime, ZonedDateTime, QueryParameter)}, but + * uses default query parameters. * - * @param session - * @param repositoryId - * @param startTime can be null - * @param endTime can be null - * @return - * @throws MetadataRepositoryException */ List getArtifactsByDateRange(RepositorySession session, String repositoryId, ZonedDateTime startTime, ZonedDateTime endTime ) throws MetadataRepositoryException; + /** + * + * Searches for artifacts where the 'whenGathered' attribute value is between the given start and end time. + * If start or end time or both are null, the time range for the search is unbounded for this parameter. + * + * + * @param session The repository session + * @param repositoryId The repository id + * @param startTime The start time/date as zoned date, can be null + * @param endTime The end time/date as zoned date, can be null + * @param queryParameter Additional parameters for the query that affect ordering and returned results + * @return The list of metadata objects for the found instances. + * @throws MetadataRepositoryException if the query fails. + * @since 3.0 + */ List getArtifactsByDateRange(RepositorySession session, String repositoryId, ZonedDateTime startTime, ZonedDateTime endTime, QueryParameter queryParameter ) throws MetadataRepositoryException; /** - * Returns all the artifacts - * @param session - * @param repositoryId - * @param startTime - * @param endTime - * @return + * Returns all the artifacts who's 'whenGathered' attribute value is inside the given time range (inclusive) as stream of objects. + * + * Implementations should return a stream of sorted objects. The objects should be sorted by the 'whenGathered' date in ascending order. + * + * @param session The repository session + * @param repositoryId The repository id + * @param startTime The start time, can be null + * @param endTime The end time, can be null + * @return A stream of artifact metadata objects. * @throws MetadataRepositoryException + * @since 3.0 */ Stream getArtifactsByDateRangeStream( RepositorySession session, String repositoryId, ZonedDateTime startTime, ZonedDateTime endTime ) throws MetadataRepositoryException; + /** + * Returns all the artifacts who's 'whenGathered' attribute value is inside the given time range (inclusive) as stream of objects. + * + * If no sort attributes are given by the queryParameter, the result is sorted by the 'whenGathered' date. + * + * @param session The repository session + * @param repositoryId The repository id + * @param startTime The start time, can be null + * @param endTime The end time, can be null + * @param queryParameter Additional parameters for the query that affect ordering and number of returned results. + * @return A stream of artifact metadata objects. + * @throws MetadataRepositoryException + * @since 3.0 + */ Stream getArtifactsByDateRangeStream(RepositorySession session, String repositoryId, ZonedDateTime startTime, ZonedDateTime endTime, QueryParameter queryParameter) throws MetadataRepositoryException; - Collection getArtifactsByChecksum( RepositorySession session, String repositoryId, String checksum ) + + /** + * Returns the artifacts that match the given checksum. All checksum types are searched. + * + * @param session The repository session + * @param repositoryId The repository id + * @param checksum The checksum as string of numbers + * @return The list of artifacts that match the given checksum. + * @throws MetadataRepositoryException + */ + List getArtifactsByChecksum(RepositorySession session, String repositoryId, String checksum ) throws MetadataRepositoryException; /** diff --git a/archiva-modules/metadata/metadata-repository-api/src/test/java/org/apache/archiva/metadata/repository/AbstractMetadataRepositoryTest.java b/archiva-modules/metadata/metadata-repository-api/src/test/java/org/apache/archiva/metadata/repository/AbstractMetadataRepositoryTest.java index d34f8ab0c..a747b7f28 100644 --- a/archiva-modules/metadata/metadata-repository-api/src/test/java/org/apache/archiva/metadata/repository/AbstractMetadataRepositoryTest.java +++ b/archiva-modules/metadata/metadata-repository-api/src/test/java/org/apache/archiva/metadata/repository/AbstractMetadataRepositoryTest.java @@ -903,7 +903,7 @@ public abstract class AbstractMetadataRepositoryTest { for (int i = 0; i<500; i++) { - getRepository( ).addMetadataFacet( session, TEST_REPO_ID, new TestMetadataFacet( TEST_FACET_ID, TEST_VALUE, TEST_NAME+"/"+i ) ); + getRepository( ).addMetadataFacet( session, TEST_REPO_ID, new TestMetadataFacet( TEST_FACET_ID, TEST_VALUE, TEST_NAME+"/"+String.format("%03d",i) ) ); } } @@ -916,14 +916,78 @@ public abstract class AbstractMetadataRepositoryTest assertEquals( 100, result.size( ) ); assertNotNull( result.get( 0 ) ); for (int i=0; i<10; i++) { - log.info( "Result {}", result.get( i ).getName( ) ); + assertEquals(TEST_NAME + "/" + String.format("%03d",i), result.get(i).getName()); } - assertEquals( TEST_NAME+"/"+0, result.get( 0 ).getName( ) ); - }, 2, 500 ); + }, 3, 500 ); } } + @Test + public void testGetMetadataFacetsStreamWithOffset( ) + throws Exception + { + try ( RepositorySession session = getSessionFactory( ).createSession( ) ) + { + for (int i = 0; i<100; i++) + { + getRepository( ).addMetadataFacet( session, TEST_REPO_ID, new TestMetadataFacet( TEST_FACET_ID, TEST_VALUE, TEST_NAME+"/"+String.format("%03d", i) ) ); + } + } + + try ( RepositorySession session = getSessionFactory( ).createSession( ) ) + { + tryAssert( ( ) -> { + Stream str = getRepository( ).getMetadataFacetStream( session, TEST_REPO_ID, TestMetadataFacet.class, new QueryParameter(5, 10)); + assertNotNull( str ); + List result = str.collect( Collectors.toList( ) ); + assertEquals( 10, result.size( ) ); + assertNotNull( result.get( 0 ) ); + for (int i=0; i<10; i++) { + assertEquals(TEST_NAME + "/" + String.format("%03d",i+5), result.get(i).getName()); + } + }, 3, 500 ); + + } + } + + @Test + public void testGetArtifactsByDateRangeStreamLowerAndUpperBound( ) + throws Exception + { + try ( RepositorySession session = getSessionFactory( ).createSession( ) ) + { + final ArtifactMetadata artifact1 = createArtifact( ); + artifact1.setWhenGathered(ZonedDateTime.now().minusDays(1)); + getRepository( ).updateArtifact( session, TEST_REPO_ID, TEST_NAMESPACE, TEST_PROJECT, TEST_PROJECT_VERSION, artifact1 ); + final ArtifactMetadata artifact2 = createArtifact( ); + artifact2.setId(artifact2.getId()+"-2"); + artifact2.setVersion(TEST_PROJECT_VERSION+"-2"); + artifact2.setWhenGathered(ZonedDateTime.now()); + getRepository( ).updateArtifact( session, TEST_REPO_ID, TEST_NAMESPACE, TEST_PROJECT, TEST_PROJECT_VERSION, artifact2 ); + final ArtifactMetadata artifact3 = createArtifact(); + artifact3.setId(artifact3.getId()+"-3"); + artifact3.setVersion(TEST_PROJECT_VERSION+"-3"); + artifact3.setWhenGathered(ZonedDateTime.now().plusDays(1)); + getRepository( ).updateArtifact( session, TEST_REPO_ID, TEST_NAMESPACE, TEST_PROJECT, TEST_PROJECT_VERSION, artifact3 ); + session.save( ); + + ZonedDateTime lower = artifact2.getWhenGathered().minusSeconds(10); + ZonedDateTime upper = artifact2.getWhenGathered().plusSeconds(10); + + tryAssert( ( ) -> { + Stream stream = getRepository().getArtifactsByDateRangeStream(session, TEST_REPO_ID, lower, upper, new QueryParameter()); + assertNotNull(stream); + + List artifacts = stream.collect(Collectors.toList()); + assertEquals(1, artifacts.size()); + + + assertEquals( Collections.singletonList( artifact2 ), artifacts ); + } ); + } + } + @Test public void testGetMetadataFacetsWhenEmpty( ) throws Exception -- cgit v1.2.3