From c424149d1cd77dc39b4a87388afabab44fce69f4 Mon Sep 17 00:00:00 2001 From: Martin Stockhammer Date: Fri, 30 Mar 2018 19:35:16 +0200 Subject: [PATCH] Updating index merge to new maven api --- .../maven/merger/DefaultIndexMerger.java | 84 ++++++++++++------- .../maven/search/MavenRepositorySearch.java | 58 ++++++------- .../archiva-rest-services/pom.xml | 5 ++ ...DownloadMergedIndexNonDefaultPathTest.java | 17 ++-- 4 files changed, 98 insertions(+), 66 deletions(-) diff --git a/archiva-modules/archiva-base/archiva-maven2-indexer/src/main/java/org/apache/archiva/indexer/maven/merger/DefaultIndexMerger.java b/archiva-modules/archiva-base/archiva-maven2-indexer/src/main/java/org/apache/archiva/indexer/maven/merger/DefaultIndexMerger.java index 0e0cd9059..1f4648025 100644 --- a/archiva-modules/archiva-base/archiva-maven2-indexer/src/main/java/org/apache/archiva/indexer/maven/merger/DefaultIndexMerger.java +++ b/archiva-modules/archiva-base/archiva-maven2-indexer/src/main/java/org/apache/archiva/indexer/maven/merger/DefaultIndexMerger.java @@ -19,15 +19,19 @@ package org.apache.archiva.indexer.maven.merger; */ import org.apache.archiva.common.utils.FileUtils; +import org.apache.archiva.indexer.UnsupportedBaseContextException; import org.apache.archiva.indexer.merger.IndexMerger; import org.apache.archiva.indexer.merger.IndexMergerException; import org.apache.archiva.indexer.merger.IndexMergerRequest; import org.apache.archiva.indexer.merger.TemporaryGroupIndex; +import org.apache.archiva.repository.RepositoryRegistry; +import org.apache.archiva.repository.RepositoryType; import org.apache.commons.lang.time.StopWatch; -import org.apache.maven.index.NexusIndexer; +import org.apache.maven.index.Indexer; +import org.apache.maven.index.context.ContextMemberProvider; import org.apache.maven.index.context.IndexCreator; import org.apache.maven.index.context.IndexingContext; -import org.apache.maven.index.context.UnsupportedExistingLuceneIndexException; +import org.apache.maven.index.context.StaticContextMemberProvider; import org.apache.maven.index.packer.IndexPacker; import org.apache.maven.index.packer.IndexPackingRequest; import org.slf4j.Logger; @@ -41,7 +45,10 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.Collection; import java.util.List; +import java.util.Objects; +import java.util.Optional; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.stream.Collectors; /** * @author Olivier Lamy @@ -52,22 +59,28 @@ public class DefaultIndexMerger implements IndexMerger { + @Inject + RepositoryRegistry repositoryRegistry; + private Logger log = LoggerFactory.getLogger( getClass() ); - private final NexusIndexer indexer; private final IndexPacker indexPacker; + private Indexer indexer; + private final List indexCreators; private List temporaryGroupIndexes = new CopyOnWriteArrayList<>(); + private List temporaryContextes = new CopyOnWriteArrayList<>( ); + private List runningGroups = new CopyOnWriteArrayList<>(); @Inject - public DefaultIndexMerger( NexusIndexer nexusIndexer, IndexPacker indexPacker, List indexCreators ) + public DefaultIndexMerger( Indexer indexer, IndexPacker indexPacker, List indexCreators ) { - this.indexer = nexusIndexer; + this.indexer = indexer; this.indexPacker = indexPacker; this.indexCreators = indexCreators; } @@ -97,25 +110,29 @@ public class DefaultIndexMerger try { Path indexLocation = mergedIndexDirectory.resolve( indexMergerRequest.getMergedIndexPath() ); - IndexingContext indexingContext = - indexer.addIndexingContext( tempRepoId, tempRepoId, mergedIndexDirectory.toFile(), indexLocation.toFile(), null, null, - indexCreators ); - for ( String repoId : indexMergerRequest.getRepositoriesIds() ) - { - IndexingContext idxToMerge = indexer.getIndexingContexts().get( repoId ); - if ( idxToMerge != null ) - { - indexingContext.merge( idxToMerge.getIndexDirectory() ); - } - } - - indexingContext.optimize(); + List members = indexMergerRequest.getRepositoriesIds( ).stream( ).map( id -> + repositoryRegistry.getRepository( id ) ).filter( repo -> repo.getType().equals( RepositoryType.MAVEN ) ) + .map( repo -> { + try + { + return repo.getIndexingContext().getBaseContext( IndexingContext.class ); + } + catch ( UnsupportedBaseContextException e ) + { + return null; + // Ignore + } + } ).filter( Objects::nonNull ).collect( Collectors.toList() ); + ContextMemberProvider memberProvider = new StaticContextMemberProvider(members); + IndexingContext mergedCtx = indexer.createMergedIndexingContext( tempRepoId, tempRepoId, mergedIndexDirectory.toFile(), + indexLocation.toFile(), true, memberProvider); + mergedCtx.optimize(); if ( indexMergerRequest.isPackIndex() ) { - IndexPackingRequest request = new IndexPackingRequest( indexingContext, // - indexingContext.acquireIndexSearcher().getIndexReader(), // + IndexPackingRequest request = new IndexPackingRequest( mergedCtx, // + mergedCtx.acquireIndexSearcher().getIndexReader(), // indexLocation.toFile() ); indexPacker.packIndex( request ); } @@ -124,13 +141,14 @@ public class DefaultIndexMerger { temporaryGroupIndexes.add( new TemporaryGroupIndex( mergedIndexDirectory, tempRepoId, groupId, indexMergerRequest.getMergedIndexTtl() ) ); + temporaryContextes.add(mergedCtx); } stopWatch.stop(); log.info( "merged index for repos {} in {} s", indexMergerRequest.getRepositoriesIds(), stopWatch.getTime() ); - return indexingContext; + return mergedCtx; } - catch ( IOException | UnsupportedExistingLuceneIndexException e ) + catch ( IOException e) { throw new IndexMergerException( e.getMessage(), e ); } @@ -151,17 +169,19 @@ public class DefaultIndexMerger try { - IndexingContext indexingContext = indexer.getIndexingContexts().get( temporaryGroupIndex.getIndexId() ); - if ( indexingContext != null ) - { - indexer.removeIndexingContext( indexingContext, true ); - } - Path directory = temporaryGroupIndex.getDirectory(); - if ( directory != null && Files.exists(directory) ) - { - FileUtils.deleteDirectory( directory ); + + Optional ctxOpt = temporaryContextes.stream( ).filter( ctx -> ctx.getId( ).equals( temporaryGroupIndex.getIndexId( ) ) ).findFirst( ); + if (ctxOpt.isPresent()) { + IndexingContext ctx = ctxOpt.get(); + indexer.closeIndexingContext( ctx, true ); + temporaryGroupIndexes.remove( temporaryGroupIndex ); + temporaryContextes.remove( ctx ); + Path directory = temporaryGroupIndex.getDirectory(); + if ( directory != null && Files.exists(directory) ) + { + FileUtils.deleteDirectory( directory ); + } } - temporaryGroupIndexes.remove( temporaryGroupIndex ); } catch ( IOException e ) { diff --git a/archiva-modules/archiva-base/archiva-maven2-indexer/src/main/java/org/apache/archiva/indexer/maven/search/MavenRepositorySearch.java b/archiva-modules/archiva-base/archiva-maven2-indexer/src/main/java/org/apache/archiva/indexer/maven/search/MavenRepositorySearch.java index 4ae252960..e18e10115 100644 --- a/archiva-modules/archiva-base/archiva-maven2-indexer/src/main/java/org/apache/archiva/indexer/maven/search/MavenRepositorySearch.java +++ b/archiva-modules/archiva-base/archiva-maven2-indexer/src/main/java/org/apache/archiva/indexer/maven/search/MavenRepositorySearch.java @@ -42,6 +42,7 @@ import org.apache.maven.index.expr.UserInputSearchExpression; import org.apache.maven.index_shaded.lucene.search.BooleanClause; import org.apache.maven.index_shaded.lucene.search.BooleanClause.Occur; import org.apache.maven.index_shaded.lucene.search.BooleanQuery; +import org.apache.maven.index_shaded.lucene.search.Query; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; @@ -98,30 +99,30 @@ public class MavenRepositorySearch // since upgrade to nexus 2.0.0, query has changed from g:[QUERIED TERM]* to g:*[QUERIED TERM]* // resulting to more wildcard searches so we need to increase max clause count BooleanQuery.setMaxClauseCount( Integer.MAX_VALUE ); - BooleanQuery q = new BooleanQuery(); + BooleanQuery.Builder qb = new BooleanQuery.Builder(); if ( previousSearchTerms == null || previousSearchTerms.isEmpty() ) { - constructQuery( term, q ); + constructQuery( term, qb ); } else { for ( String previousTerm : previousSearchTerms ) { - BooleanQuery iQuery = new BooleanQuery(); + BooleanQuery.Builder iQuery = new BooleanQuery.Builder(); constructQuery( previousTerm, iQuery ); - q.add( iQuery, BooleanClause.Occur.MUST ); + qb.add( iQuery.build(), BooleanClause.Occur.MUST ); } - BooleanQuery iQuery = new BooleanQuery(); + BooleanQuery.Builder iQuery = new BooleanQuery.Builder(); constructQuery( term, iQuery ); - q.add( iQuery, BooleanClause.Occur.MUST ); + qb.add( iQuery.build(), BooleanClause.Occur.MUST ); } // we retun only artifacts without classifier in quick search, olamy cannot find a way to say with this field empty // FIXME cannot find a way currently to setup this in constructQuery !!! - return search( limits, q, indexingContextIds, NoClassifierArtifactInfoFilter.LIST, selectedRepos, true ); + return search( limits, qb.build(), indexingContextIds, NoClassifierArtifactInfoFilter.LIST, selectedRepos, true ); } @@ -147,17 +148,17 @@ public class MavenRepositorySearch return new SearchResults(); } - BooleanQuery q = new BooleanQuery(); + BooleanQuery.Builder qb = new BooleanQuery.Builder(); if ( StringUtils.isNotBlank( searchFields.getGroupId() ) ) { - q.add( indexer.constructQuery( MAVEN.GROUP_ID, searchFields.isExactSearch() ? new SourcedSearchExpression( + qb.add( indexer.constructQuery( MAVEN.GROUP_ID, searchFields.isExactSearch() ? new SourcedSearchExpression( searchFields.getGroupId() ) : new UserInputSearchExpression( searchFields.getGroupId() ) ), BooleanClause.Occur.MUST ); } if ( StringUtils.isNotBlank( searchFields.getArtifactId() ) ) { - q.add( indexer.constructQuery( MAVEN.ARTIFACT_ID, + qb.add( indexer.constructQuery( MAVEN.ARTIFACT_ID, searchFields.isExactSearch() ? new SourcedSearchExpression( searchFields.getArtifactId() ) : new UserInputSearchExpression( searchFields.getArtifactId() ) ), @@ -166,83 +167,83 @@ public class MavenRepositorySearch if ( StringUtils.isNotBlank( searchFields.getVersion() ) ) { - q.add( indexer.constructQuery( MAVEN.VERSION, searchFields.isExactSearch() ? new SourcedSearchExpression( + qb.add( indexer.constructQuery( MAVEN.VERSION, searchFields.isExactSearch() ? new SourcedSearchExpression( searchFields.getVersion() ) : new SourcedSearchExpression( searchFields.getVersion() ) ), BooleanClause.Occur.MUST ); } if ( StringUtils.isNotBlank( searchFields.getPackaging() ) ) { - q.add( indexer.constructQuery( MAVEN.PACKAGING, searchFields.isExactSearch() ? new SourcedSearchExpression( + qb.add( indexer.constructQuery( MAVEN.PACKAGING, searchFields.isExactSearch() ? new SourcedSearchExpression( searchFields.getPackaging() ) : new UserInputSearchExpression( searchFields.getPackaging() ) ), BooleanClause.Occur.MUST ); } if ( StringUtils.isNotBlank( searchFields.getClassName() ) ) { - q.add( indexer.constructQuery( MAVEN.CLASSNAMES, + qb.add( indexer.constructQuery( MAVEN.CLASSNAMES, new UserInputSearchExpression( searchFields.getClassName() ) ), BooleanClause.Occur.MUST ); } if ( StringUtils.isNotBlank( searchFields.getBundleSymbolicName() ) ) { - q.add( indexer.constructQuery( OSGI.SYMBOLIC_NAME, + qb.add( indexer.constructQuery( OSGI.SYMBOLIC_NAME, new UserInputSearchExpression( searchFields.getBundleSymbolicName() ) ), BooleanClause.Occur.MUST ); } if ( StringUtils.isNotBlank( searchFields.getBundleVersion() ) ) { - q.add( indexer.constructQuery( OSGI.VERSION, + qb.add( indexer.constructQuery( OSGI.VERSION, new UserInputSearchExpression( searchFields.getBundleVersion() ) ), BooleanClause.Occur.MUST ); } if ( StringUtils.isNotBlank( searchFields.getBundleExportPackage() ) ) { - q.add( indexer.constructQuery( OSGI.EXPORT_PACKAGE, + qb.add( indexer.constructQuery( OSGI.EXPORT_PACKAGE, new UserInputSearchExpression( searchFields.getBundleExportPackage() ) ), Occur.MUST ); } if ( StringUtils.isNotBlank( searchFields.getBundleExportService() ) ) { - q.add( indexer.constructQuery( OSGI.EXPORT_SERVICE, + qb.add( indexer.constructQuery( OSGI.EXPORT_SERVICE, new UserInputSearchExpression( searchFields.getBundleExportService() ) ), Occur.MUST ); } if ( StringUtils.isNotBlank( searchFields.getBundleImportPackage() ) ) { - q.add( indexer.constructQuery( OSGI.IMPORT_PACKAGE, + qb.add( indexer.constructQuery( OSGI.IMPORT_PACKAGE, new UserInputSearchExpression( searchFields.getBundleImportPackage() ) ), Occur.MUST ); } if ( StringUtils.isNotBlank( searchFields.getBundleName() ) ) { - q.add( indexer.constructQuery( OSGI.NAME, new UserInputSearchExpression( searchFields.getBundleName() ) ), + qb.add( indexer.constructQuery( OSGI.NAME, new UserInputSearchExpression( searchFields.getBundleName() ) ), Occur.MUST ); } if ( StringUtils.isNotBlank( searchFields.getBundleImportPackage() ) ) { - q.add( indexer.constructQuery( OSGI.IMPORT_PACKAGE, + qb.add( indexer.constructQuery( OSGI.IMPORT_PACKAGE, new UserInputSearchExpression( searchFields.getBundleImportPackage() ) ), Occur.MUST ); } if ( StringUtils.isNotBlank( searchFields.getBundleRequireBundle() ) ) { - q.add( indexer.constructQuery( OSGI.REQUIRE_BUNDLE, + qb.add( indexer.constructQuery( OSGI.REQUIRE_BUNDLE, new UserInputSearchExpression( searchFields.getBundleRequireBundle() ) ), Occur.MUST ); } if ( StringUtils.isNotBlank( searchFields.getClassifier() ) ) { - q.add( indexer.constructQuery( MAVEN.CLASSIFIER, searchFields.isExactSearch() ? new SourcedSearchExpression( + qb.add( indexer.constructQuery( MAVEN.CLASSIFIER, searchFields.isExactSearch() ? new SourcedSearchExpression( searchFields.getClassifier() ) : new UserInputSearchExpression( searchFields.getClassifier() ) ), Occur.MUST ); } @@ -252,18 +253,19 @@ public class MavenRepositorySearch // currently it's done in DefaultSearchService with some filtering } - if ( q.getClauses() == null || q.getClauses().length <= 0 ) + BooleanQuery qu = qb.build(); + if ( qu.clauses() == null || qu.clauses().size() <= 0 ) { throw new RepositorySearchException( "No search fields set." ); } - if (q.getClauses()!=null) { - log.debug("CLAUSES ", q.getClauses()); - for (BooleanClause cl : q.getClauses()) { + if (qu.clauses()!=null) { + log.debug("CLAUSES ", qu.clauses()); + for (BooleanClause cl : qu.clauses()) { log.debug("Clause ",cl); } } - return search( limits, q, indexingContextIds, Collections.emptyList(), + return search( limits, qu, indexingContextIds, Collections.emptyList(), searchFields.getRepositories(), searchFields.isIncludePomArtifacts() ); } @@ -370,7 +372,7 @@ public class MavenRepositorySearch return contexts; } - private void constructQuery( String term, BooleanQuery q ) + private void constructQuery( String term, BooleanQuery.Builder q ) { q.add( indexer.constructQuery( MAVEN.GROUP_ID, new UserInputSearchExpression( term ) ), Occur.SHOULD ); q.add( indexer.constructQuery( MAVEN.ARTIFACT_ID, new UserInputSearchExpression( term ) ), Occur.SHOULD ); diff --git a/archiva-modules/archiva-web/archiva-rest/archiva-rest-services/pom.xml b/archiva-modules/archiva-web/archiva-rest/archiva-rest-services/pom.xml index 28c507e60..58b1a0322 100644 --- a/archiva-modules/archiva-web/archiva-rest/archiva-rest-services/pom.xml +++ b/archiva-modules/archiva-web/archiva-rest/archiva-rest-services/pom.xml @@ -96,6 +96,11 @@ archiva-maven2-model + + org.apache.archiva + archiva-model + + org.jsoup jsoup diff --git a/archiva-modules/archiva-web/archiva-web-common/src/test/java/org/apache/archiva/remotedownload/DownloadMergedIndexNonDefaultPathTest.java b/archiva-modules/archiva-web/archiva-web-common/src/test/java/org/apache/archiva/remotedownload/DownloadMergedIndexNonDefaultPathTest.java index 95f56ab1d..d2c569c95 100644 --- a/archiva-modules/archiva-web/archiva-web-common/src/test/java/org/apache/archiva/remotedownload/DownloadMergedIndexNonDefaultPathTest.java +++ b/archiva-modules/archiva-web/archiva-web-common/src/test/java/org/apache/archiva/remotedownload/DownloadMergedIndexNonDefaultPathTest.java @@ -22,6 +22,7 @@ import org.apache.archiva.admin.model.beans.ManagedRepository; import org.apache.archiva.admin.model.beans.ProxyConnector; import org.apache.archiva.admin.model.beans.RemoteRepository; import org.apache.archiva.admin.model.beans.RepositoryGroup; +import org.apache.archiva.common.utils.FileUtils; import org.apache.archiva.maven2.model.Artifact; import org.apache.archiva.redback.integration.security.role.RedbackRoleConstants; import org.apache.archiva.redback.rest.services.FakeCreateAdminService; @@ -87,17 +88,20 @@ public class DownloadMergedIndexNonDefaultPathTest public void downloadMergedIndexWithNonDefaultPath() throws Exception { - Path tmpIndexDir = Paths.get( System.getProperty( "java.io.tmpdir" ), "tmpIndex" ); - if ( Files.exists(tmpIndexDir) ) - { - org.apache.archiva.common.utils.FileUtils.deleteDirectory( tmpIndexDir ); + + Path indexBaseDir = Paths.get(System.getProperty( "java.io.tmpdir" )).resolve("archiva").resolve("remotedownloadtest"); + String indexBase = indexBaseDir.toString(); + FileUtils.deleteQuietly( indexBaseDir); + if (!Files.exists(indexBaseDir)) { + Files.createDirectories( indexBaseDir ); } String id = Long.toString( System.currentTimeMillis() ); ManagedRepository managedRepository = new ManagedRepository( Locale.getDefault()); managedRepository.setId( id ); managedRepository.setName( "name of " + id ); managedRepository.setLocation( System.getProperty( "basedir" ) + "/src/test/repositories/test-repo" ); - managedRepository.setIndexDirectory( System.getProperty( "java.io.tmpdir" ) + "/tmpIndex/" + id ); + managedRepository.setIndexDirectory( indexBase + "/index-" + id ); + managedRepository.setPackedIndexDirectory( indexBase + "/indexPacked-" + id ); ManagedRepositoriesService managedRepositoriesService = getManagedRepositoriesService(); @@ -143,7 +147,8 @@ public class DownloadMergedIndexNonDefaultPathTest managedRepository.setId( id ); managedRepository.setName( "name of " + id ); managedRepository.setLocation( System.getProperty( "basedir" ) + "/src/test/repositories/test-repo" ); - managedRepository.setIndexDirectory( System.getProperty( "java.io.tmpdir" ) + "/tmpIndex/" + id ); + managedRepository.setIndexDirectory( indexBaseDir + "/index-"+ id ); + managedRepository.setPackedIndexDirectory( indexBase + "/tmpIndexPacked-" + id ); if ( managedRepositoriesService.getManagedRepository( id ) != null ) { -- 2.39.5