mirrors
/
archiva
kopie van https://github.com/apache/archiva.git
Volgen 1
Ster 0
Vork 0
Code Kwesties 0 Publicaties 70 Wiki Activiteit
Bladeren bron

Updating index merge to new maven api

pull/46/head
Martin Stockhammer 6 jaren geleden
bovenliggende
8cb4ef0aea
commit
c424149d1c
4 gewijzigde bestanden met toevoegingen van 98 en 66 verwijderingen
Zij-aan-zij weergave Toon Diff Stats
  1. 52
    32
      archiva-modules/archiva-base/archiva-maven2-indexer/src/main/java/org/apache/archiva/indexer/maven/merger/DefaultIndexMerger.java
  2. 30
    28
      archiva-modules/archiva-base/archiva-maven2-indexer/src/main/java/org/apache/archiva/indexer/maven/search/MavenRepositorySearch.java
  3. 5
    0
      archiva-modules/archiva-web/archiva-rest/archiva-rest-services/pom.xml
  4. 11
    6
      archiva-modules/archiva-web/archiva-web-common/src/test/java/org/apache/archiva/remotedownload/DownloadMergedIndexNonDefaultPathTest.java

+ 52
- 32
archiva-modules/archiva-base/archiva-maven2-indexer/src/main/java/org/apache/archiva/indexer/maven/merger/DefaultIndexMerger.java Bestand weergeven

@@ -19,15 +19,19 @@ package org.apache.archiva.indexer.maven.merger;
*/

import org.apache.archiva.common.utils.FileUtils;
import org.apache.archiva.indexer.UnsupportedBaseContextException;
import org.apache.archiva.indexer.merger.IndexMerger;
import org.apache.archiva.indexer.merger.IndexMergerException;
import org.apache.archiva.indexer.merger.IndexMergerRequest;
import org.apache.archiva.indexer.merger.TemporaryGroupIndex;
import org.apache.archiva.repository.RepositoryRegistry;
import org.apache.archiva.repository.RepositoryType;
import org.apache.commons.lang.time.StopWatch;
import org.apache.maven.index.NexusIndexer;
import org.apache.maven.index.Indexer;
import org.apache.maven.index.context.ContextMemberProvider;
import org.apache.maven.index.context.IndexCreator;
import org.apache.maven.index.context.IndexingContext;
import org.apache.maven.index.context.UnsupportedExistingLuceneIndexException;
import org.apache.maven.index.context.StaticContextMemberProvider;
import org.apache.maven.index.packer.IndexPacker;
import org.apache.maven.index.packer.IndexPackingRequest;
import org.slf4j.Logger;
@@ -41,7 +45,10 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collection;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.stream.Collectors;

/**
* @author Olivier Lamy
@@ -52,22 +59,28 @@ public class DefaultIndexMerger
implements IndexMerger
{

@Inject
RepositoryRegistry repositoryRegistry;

private Logger log = LoggerFactory.getLogger( getClass() );

private final NexusIndexer indexer;

private final IndexPacker indexPacker;

private Indexer indexer;

private final List<IndexCreator> indexCreators;

private List<TemporaryGroupIndex> temporaryGroupIndexes = new CopyOnWriteArrayList<>();

private List<IndexingContext> temporaryContextes = new CopyOnWriteArrayList<>( );

private List<String> runningGroups = new CopyOnWriteArrayList<>();

@Inject
public DefaultIndexMerger( NexusIndexer nexusIndexer, IndexPacker indexPacker, List<IndexCreator> indexCreators )
public DefaultIndexMerger( Indexer indexer, IndexPacker indexPacker, List<IndexCreator> indexCreators )
{
this.indexer = nexusIndexer;
this.indexer = indexer;
this.indexPacker = indexPacker;
this.indexCreators = indexCreators;
}
@@ -97,25 +110,29 @@ public class DefaultIndexMerger
try
{
Path indexLocation = mergedIndexDirectory.resolve( indexMergerRequest.getMergedIndexPath() );
IndexingContext indexingContext =
indexer.addIndexingContext( tempRepoId, tempRepoId, mergedIndexDirectory.toFile(), indexLocation.toFile(), null, null,
indexCreators );

for ( String repoId : indexMergerRequest.getRepositoriesIds() )
{
IndexingContext idxToMerge = indexer.getIndexingContexts().get( repoId );
if ( idxToMerge != null )
{
indexingContext.merge( idxToMerge.getIndexDirectory() );
}
}

indexingContext.optimize();
List<IndexingContext> members = indexMergerRequest.getRepositoriesIds( ).stream( ).map( id ->
repositoryRegistry.getRepository( id ) ).filter( repo -> repo.getType().equals( RepositoryType.MAVEN ) )
.map( repo -> {
try
{
return repo.getIndexingContext().getBaseContext( IndexingContext.class );
}
catch ( UnsupportedBaseContextException e )
{
return null;
// Ignore
}
} ).filter( Objects::nonNull ).collect( Collectors.toList() );
ContextMemberProvider memberProvider = new StaticContextMemberProvider(members);
IndexingContext mergedCtx = indexer.createMergedIndexingContext( tempRepoId, tempRepoId, mergedIndexDirectory.toFile(),
indexLocation.toFile(), true, memberProvider);
mergedCtx.optimize();

if ( indexMergerRequest.isPackIndex() )
{
IndexPackingRequest request = new IndexPackingRequest( indexingContext, //
indexingContext.acquireIndexSearcher().getIndexReader(), //
IndexPackingRequest request = new IndexPackingRequest( mergedCtx, //
mergedCtx.acquireIndexSearcher().getIndexReader(), //
indexLocation.toFile() );
indexPacker.packIndex( request );
}
@@ -124,13 +141,14 @@ public class DefaultIndexMerger
{
temporaryGroupIndexes.add( new TemporaryGroupIndex( mergedIndexDirectory, tempRepoId, groupId,
indexMergerRequest.getMergedIndexTtl() ) );
temporaryContextes.add(mergedCtx);
}
stopWatch.stop();
log.info( "merged index for repos {} in {} s", indexMergerRequest.getRepositoriesIds(),
stopWatch.getTime() );
return indexingContext;
return mergedCtx;
}
catch ( IOException | UnsupportedExistingLuceneIndexException e )
catch ( IOException e)
{
throw new IndexMergerException( e.getMessage(), e );
}
@@ -151,17 +169,19 @@ public class DefaultIndexMerger

try
{
IndexingContext indexingContext = indexer.getIndexingContexts().get( temporaryGroupIndex.getIndexId() );
if ( indexingContext != null )
{
indexer.removeIndexingContext( indexingContext, true );
}
Path directory = temporaryGroupIndex.getDirectory();
if ( directory != null && Files.exists(directory) )
{
FileUtils.deleteDirectory( directory );

Optional<IndexingContext> ctxOpt = temporaryContextes.stream( ).filter( ctx -> ctx.getId( ).equals( temporaryGroupIndex.getIndexId( ) ) ).findFirst( );
if (ctxOpt.isPresent()) {
IndexingContext ctx = ctxOpt.get();
indexer.closeIndexingContext( ctx, true );
temporaryGroupIndexes.remove( temporaryGroupIndex );
temporaryContextes.remove( ctx );
Path directory = temporaryGroupIndex.getDirectory();
if ( directory != null && Files.exists(directory) )
{
FileUtils.deleteDirectory( directory );
}
}
temporaryGroupIndexes.remove( temporaryGroupIndex );
}
catch ( IOException e )
{

+ 30
- 28
archiva-modules/archiva-base/archiva-maven2-indexer/src/main/java/org/apache/archiva/indexer/maven/search/MavenRepositorySearch.java Bestand weergeven

@@ -42,6 +42,7 @@ import org.apache.maven.index.expr.UserInputSearchExpression;
import org.apache.maven.index_shaded.lucene.search.BooleanClause;
import org.apache.maven.index_shaded.lucene.search.BooleanClause.Occur;
import org.apache.maven.index_shaded.lucene.search.BooleanQuery;
import org.apache.maven.index_shaded.lucene.search.Query;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
@@ -98,30 +99,30 @@ public class MavenRepositorySearch
// since upgrade to nexus 2.0.0, query has changed from g:[QUERIED TERM]* to g:*[QUERIED TERM]*
// resulting to more wildcard searches so we need to increase max clause count
BooleanQuery.setMaxClauseCount( Integer.MAX_VALUE );
BooleanQuery q = new BooleanQuery();
BooleanQuery.Builder qb = new BooleanQuery.Builder();

if ( previousSearchTerms == null || previousSearchTerms.isEmpty() )
{
constructQuery( term, q );
constructQuery( term, qb );
}
else
{
for ( String previousTerm : previousSearchTerms )
{
BooleanQuery iQuery = new BooleanQuery();
BooleanQuery.Builder iQuery = new BooleanQuery.Builder();
constructQuery( previousTerm, iQuery );

q.add( iQuery, BooleanClause.Occur.MUST );
qb.add( iQuery.build(), BooleanClause.Occur.MUST );
}

BooleanQuery iQuery = new BooleanQuery();
BooleanQuery.Builder iQuery = new BooleanQuery.Builder();
constructQuery( term, iQuery );
q.add( iQuery, BooleanClause.Occur.MUST );
qb.add( iQuery.build(), BooleanClause.Occur.MUST );
}

// we retun only artifacts without classifier in quick search, olamy cannot find a way to say with this field empty
// FIXME cannot find a way currently to setup this in constructQuery !!!
return search( limits, q, indexingContextIds, NoClassifierArtifactInfoFilter.LIST, selectedRepos, true );
return search( limits, qb.build(), indexingContextIds, NoClassifierArtifactInfoFilter.LIST, selectedRepos, true );

}

@@ -147,17 +148,17 @@ public class MavenRepositorySearch
return new SearchResults();
}

BooleanQuery q = new BooleanQuery();
BooleanQuery.Builder qb = new BooleanQuery.Builder();
if ( StringUtils.isNotBlank( searchFields.getGroupId() ) )
{
q.add( indexer.constructQuery( MAVEN.GROUP_ID, searchFields.isExactSearch() ? new SourcedSearchExpression(
qb.add( indexer.constructQuery( MAVEN.GROUP_ID, searchFields.isExactSearch() ? new SourcedSearchExpression(
searchFields.getGroupId() ) : new UserInputSearchExpression( searchFields.getGroupId() ) ),
BooleanClause.Occur.MUST );
}

if ( StringUtils.isNotBlank( searchFields.getArtifactId() ) )
{
q.add( indexer.constructQuery( MAVEN.ARTIFACT_ID,
qb.add( indexer.constructQuery( MAVEN.ARTIFACT_ID,
searchFields.isExactSearch()
? new SourcedSearchExpression( searchFields.getArtifactId() )
: new UserInputSearchExpression( searchFields.getArtifactId() ) ),
@@ -166,83 +167,83 @@ public class MavenRepositorySearch

if ( StringUtils.isNotBlank( searchFields.getVersion() ) )
{
q.add( indexer.constructQuery( MAVEN.VERSION, searchFields.isExactSearch() ? new SourcedSearchExpression(
qb.add( indexer.constructQuery( MAVEN.VERSION, searchFields.isExactSearch() ? new SourcedSearchExpression(
searchFields.getVersion() ) : new SourcedSearchExpression( searchFields.getVersion() ) ),
BooleanClause.Occur.MUST );
}

if ( StringUtils.isNotBlank( searchFields.getPackaging() ) )
{
q.add( indexer.constructQuery( MAVEN.PACKAGING, searchFields.isExactSearch() ? new SourcedSearchExpression(
qb.add( indexer.constructQuery( MAVEN.PACKAGING, searchFields.isExactSearch() ? new SourcedSearchExpression(
searchFields.getPackaging() ) : new UserInputSearchExpression( searchFields.getPackaging() ) ),
BooleanClause.Occur.MUST );
}

if ( StringUtils.isNotBlank( searchFields.getClassName() ) )
{
q.add( indexer.constructQuery( MAVEN.CLASSNAMES,
qb.add( indexer.constructQuery( MAVEN.CLASSNAMES,
new UserInputSearchExpression( searchFields.getClassName() ) ),
BooleanClause.Occur.MUST );
}

if ( StringUtils.isNotBlank( searchFields.getBundleSymbolicName() ) )
{
q.add( indexer.constructQuery( OSGI.SYMBOLIC_NAME,
qb.add( indexer.constructQuery( OSGI.SYMBOLIC_NAME,
new UserInputSearchExpression( searchFields.getBundleSymbolicName() ) ),
BooleanClause.Occur.MUST );
}

if ( StringUtils.isNotBlank( searchFields.getBundleVersion() ) )
{
q.add( indexer.constructQuery( OSGI.VERSION,
qb.add( indexer.constructQuery( OSGI.VERSION,
new UserInputSearchExpression( searchFields.getBundleVersion() ) ),
BooleanClause.Occur.MUST );
}

if ( StringUtils.isNotBlank( searchFields.getBundleExportPackage() ) )
{
q.add( indexer.constructQuery( OSGI.EXPORT_PACKAGE,
qb.add( indexer.constructQuery( OSGI.EXPORT_PACKAGE,
new UserInputSearchExpression( searchFields.getBundleExportPackage() ) ),
Occur.MUST );
}

if ( StringUtils.isNotBlank( searchFields.getBundleExportService() ) )
{
q.add( indexer.constructQuery( OSGI.EXPORT_SERVICE,
qb.add( indexer.constructQuery( OSGI.EXPORT_SERVICE,
new UserInputSearchExpression( searchFields.getBundleExportService() ) ),
Occur.MUST );
}

if ( StringUtils.isNotBlank( searchFields.getBundleImportPackage() ) )
{
q.add( indexer.constructQuery( OSGI.IMPORT_PACKAGE,
qb.add( indexer.constructQuery( OSGI.IMPORT_PACKAGE,
new UserInputSearchExpression( searchFields.getBundleImportPackage() ) ),
Occur.MUST );
}

if ( StringUtils.isNotBlank( searchFields.getBundleName() ) )
{
q.add( indexer.constructQuery( OSGI.NAME, new UserInputSearchExpression( searchFields.getBundleName() ) ),
qb.add( indexer.constructQuery( OSGI.NAME, new UserInputSearchExpression( searchFields.getBundleName() ) ),
Occur.MUST );
}

if ( StringUtils.isNotBlank( searchFields.getBundleImportPackage() ) )
{
q.add( indexer.constructQuery( OSGI.IMPORT_PACKAGE,
qb.add( indexer.constructQuery( OSGI.IMPORT_PACKAGE,
new UserInputSearchExpression( searchFields.getBundleImportPackage() ) ),
Occur.MUST );
}

if ( StringUtils.isNotBlank( searchFields.getBundleRequireBundle() ) )
{
q.add( indexer.constructQuery( OSGI.REQUIRE_BUNDLE,
qb.add( indexer.constructQuery( OSGI.REQUIRE_BUNDLE,
new UserInputSearchExpression( searchFields.getBundleRequireBundle() ) ),
Occur.MUST );
}

if ( StringUtils.isNotBlank( searchFields.getClassifier() ) )
{
q.add( indexer.constructQuery( MAVEN.CLASSIFIER, searchFields.isExactSearch() ? new SourcedSearchExpression(
qb.add( indexer.constructQuery( MAVEN.CLASSIFIER, searchFields.isExactSearch() ? new SourcedSearchExpression(
searchFields.getClassifier() ) : new UserInputSearchExpression( searchFields.getClassifier() ) ),
Occur.MUST );
}
@@ -252,18 +253,19 @@ public class MavenRepositorySearch
// currently it's done in DefaultSearchService with some filtering
}

if ( q.getClauses() == null || q.getClauses().length <= 0 )
BooleanQuery qu = qb.build();
if ( qu.clauses() == null || qu.clauses().size() <= 0 )
{
throw new RepositorySearchException( "No search fields set." );
}
if (q.getClauses()!=null) {
log.debug("CLAUSES ", q.getClauses());
for (BooleanClause cl : q.getClauses()) {
if (qu.clauses()!=null) {
log.debug("CLAUSES ", qu.clauses());
for (BooleanClause cl : qu.clauses()) {
log.debug("Clause ",cl);
}
}

return search( limits, q, indexingContextIds, Collections.<ArtifactInfoFilter>emptyList(),
return search( limits, qu, indexingContextIds, Collections.<ArtifactInfoFilter>emptyList(),
searchFields.getRepositories(), searchFields.isIncludePomArtifacts() );
}

@@ -370,7 +372,7 @@ public class MavenRepositorySearch
return contexts;
}

private void constructQuery( String term, BooleanQuery q )
private void constructQuery( String term, BooleanQuery.Builder q )
{
q.add( indexer.constructQuery( MAVEN.GROUP_ID, new UserInputSearchExpression( term ) ), Occur.SHOULD );
q.add( indexer.constructQuery( MAVEN.ARTIFACT_ID, new UserInputSearchExpression( term ) ), Occur.SHOULD );

+ 5
- 0
archiva-modules/archiva-web/archiva-rest/archiva-rest-services/pom.xml Bestand weergeven

@@ -96,6 +96,11 @@
<artifactId>archiva-maven2-model</artifactId>
</dependency>

<dependency>
<groupId>org.apache.archiva</groupId>
<artifactId>archiva-model</artifactId>
</dependency>

<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>

+ 11
- 6
archiva-modules/archiva-web/archiva-web-common/src/test/java/org/apache/archiva/remotedownload/DownloadMergedIndexNonDefaultPathTest.java Bestand weergeven

@@ -22,6 +22,7 @@ import org.apache.archiva.admin.model.beans.ManagedRepository;
import org.apache.archiva.admin.model.beans.ProxyConnector;
import org.apache.archiva.admin.model.beans.RemoteRepository;
import org.apache.archiva.admin.model.beans.RepositoryGroup;
import org.apache.archiva.common.utils.FileUtils;
import org.apache.archiva.maven2.model.Artifact;
import org.apache.archiva.redback.integration.security.role.RedbackRoleConstants;
import org.apache.archiva.redback.rest.services.FakeCreateAdminService;
@@ -87,17 +88,20 @@ public class DownloadMergedIndexNonDefaultPathTest
public void downloadMergedIndexWithNonDefaultPath()
throws Exception
{
Path tmpIndexDir = Paths.get( System.getProperty( "java.io.tmpdir" ), "tmpIndex" );
if ( Files.exists(tmpIndexDir) )
{
org.apache.archiva.common.utils.FileUtils.deleteDirectory( tmpIndexDir );

Path indexBaseDir = Paths.get(System.getProperty( "java.io.tmpdir" )).resolve("archiva").resolve("remotedownloadtest");
String indexBase = indexBaseDir.toString();
FileUtils.deleteQuietly( indexBaseDir);
if (!Files.exists(indexBaseDir)) {
Files.createDirectories( indexBaseDir );
}
String id = Long.toString( System.currentTimeMillis() );
ManagedRepository managedRepository = new ManagedRepository( Locale.getDefault());
managedRepository.setId( id );
managedRepository.setName( "name of " + id );
managedRepository.setLocation( System.getProperty( "basedir" ) + "/src/test/repositories/test-repo" );
managedRepository.setIndexDirectory( System.getProperty( "java.io.tmpdir" ) + "/tmpIndex/" + id );
managedRepository.setIndexDirectory( indexBase + "/index-" + id );
managedRepository.setPackedIndexDirectory( indexBase + "/indexPacked-" + id );

ManagedRepositoriesService managedRepositoriesService = getManagedRepositoriesService();

@@ -143,7 +147,8 @@ public class DownloadMergedIndexNonDefaultPathTest
managedRepository.setId( id );
managedRepository.setName( "name of " + id );
managedRepository.setLocation( System.getProperty( "basedir" ) + "/src/test/repositories/test-repo" );
managedRepository.setIndexDirectory( System.getProperty( "java.io.tmpdir" ) + "/tmpIndex/" + id );
managedRepository.setIndexDirectory( indexBaseDir + "/index-"+ id );
managedRepository.setPackedIndexDirectory( indexBase + "/tmpIndexPacked-" + id );

if ( managedRepositoriesService.getManagedRepository( id ) != null )
{

Schrijf Voorbeeld
Laden…
Annuleren
Opslaan