1 package org.apache.archiva.indexer.maven.search;
4 * Licensed to the Apache Software Foundation (ASF) under one
5 * or more contributor license agreements. See the NOTICE file
6 * distributed with this work for additional information
7 * regarding copyright ownership. The ASF licenses this file
8 * to you under the Apache License, Version 2.0 (the
9 * "License"); you may not use this file except in compliance
10 * with the License. You may obtain a copy of the License at
12 * http://www.apache.org/licenses/LICENSE-2.0
14 * Unless required by applicable law or agreed to in writing,
15 * software distributed under the License is distributed on an
16 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17 * KIND, either express or implied. See the License for the
18 * specific language governing permissions and limitations
22 import org.apache.archiva.admin.model.RepositoryAdminException;
23 import org.apache.archiva.admin.model.beans.ProxyConnector;
24 import org.apache.archiva.admin.model.proxyconnector.ProxyConnectorAdmin;
25 import org.apache.archiva.common.plexusbridge.PlexusSisuBridgeException;
26 import org.apache.archiva.indexer.UnsupportedBaseContextException;
27 import org.apache.archiva.indexer.search.ArtifactInfoFilter;
28 import org.apache.archiva.indexer.search.*;
29 import org.apache.archiva.indexer.util.SearchUtil;
30 import org.apache.archiva.model.ArchivaArtifactModel;
31 import org.apache.archiva.repository.RemoteRepository;
32 import org.apache.archiva.repository.Repository;
33 import org.apache.archiva.repository.RepositoryRegistry;
34 import org.apache.archiva.repository.RepositoryType;
35 import org.apache.commons.lang.StringUtils;
36 import org.apache.maven.index.*;
37 import org.apache.maven.index.context.IndexingContext;
38 import org.apache.maven.index.expr.SearchExpression;
39 import org.apache.maven.index.expr.SearchTyped;
40 import org.apache.maven.index.expr.SourcedSearchExpression;
41 import org.apache.maven.index.expr.UserInputSearchExpression;
42 import org.apache.maven.index_shaded.lucene.search.BooleanClause;
43 import org.apache.maven.index_shaded.lucene.search.BooleanClause.Occur;
44 import org.apache.maven.index_shaded.lucene.search.BooleanQuery;
45 import org.slf4j.Logger;
46 import org.slf4j.LoggerFactory;
47 import org.springframework.stereotype.Service;
49 import javax.inject.Inject;
50 import java.io.IOException;
54 * RepositorySearch implementation which uses the Maven Indexer for searching.
56 @Service( "repositorySearch#maven" )
57 public class MavenRepositorySearch
58 implements RepositorySearch
60 private Logger log = LoggerFactory.getLogger( getClass() );
62 private Indexer indexer;
64 private QueryCreator queryCreator;
67 RepositoryRegistry repositoryRegistry;
69 private ProxyConnectorAdmin proxyConnectorAdmin;
71 protected MavenRepositorySearch()
77 public MavenRepositorySearch( Indexer nexusIndexer, RepositoryRegistry repositoryRegistry,
79 ProxyConnectorAdmin proxyConnectorAdmin, QueryCreator queryCreator )
80 throws PlexusSisuBridgeException
82 this.indexer = nexusIndexer;
83 this.queryCreator = queryCreator;
84 this.repositoryRegistry = repositoryRegistry;
85 this.proxyConnectorAdmin = proxyConnectorAdmin;
89 * @see RepositorySearch#search(String, List, String, SearchResultLimits, List)
92 public SearchResults search(String principal, List<String> selectedRepos, String term, SearchResultLimits limits,
93 List<String> previousSearchTerms )
94 throws RepositorySearchException
96 List<String> indexingContextIds = addIndexingContexts( selectedRepos );
98 // since upgrade to nexus 2.0.0, query has changed from g:[QUERIED TERM]* to g:*[QUERIED TERM]*
99 // resulting to more wildcard searches so we need to increase max clause count
100 BooleanQuery.setMaxClauseCount( Integer.MAX_VALUE );
101 BooleanQuery q = new BooleanQuery();
103 if ( previousSearchTerms == null || previousSearchTerms.isEmpty() )
105 constructQuery( term, q );
109 for ( String previousTerm : previousSearchTerms )
111 BooleanQuery iQuery = new BooleanQuery();
112 constructQuery( previousTerm, iQuery );
114 q.add( iQuery, BooleanClause.Occur.MUST );
117 BooleanQuery iQuery = new BooleanQuery();
118 constructQuery( term, iQuery );
119 q.add( iQuery, BooleanClause.Occur.MUST );
122 // we retun only artifacts without classifier in quick search, olamy cannot find a way to say with this field empty
123 // FIXME cannot find a way currently to setup this in constructQuery !!!
124 return search( limits, q, indexingContextIds, NoClassifierArtifactInfoFilter.LIST, selectedRepos, true );
129 * @see RepositorySearch#search(String, SearchFields, SearchResultLimits)
132 public SearchResults search( String principal, SearchFields searchFields, SearchResultLimits limits )
133 throws RepositorySearchException
135 if ( searchFields.getRepositories() == null )
137 throw new RepositorySearchException( "Repositories cannot be null." );
140 List<String> indexingContextIds = addIndexingContexts( searchFields.getRepositories() );
142 // if no index found in the specified ones return an empty search result instead of doing a search on all index
143 // olamy: IMHO doesn't make sense
144 if ( !searchFields.getRepositories().isEmpty() && ( indexingContextIds == null
145 || indexingContextIds.isEmpty() ) )
147 return new SearchResults();
150 BooleanQuery q = new BooleanQuery();
151 if ( StringUtils.isNotBlank( searchFields.getGroupId() ) )
153 q.add( indexer.constructQuery( MAVEN.GROUP_ID, searchFields.isExactSearch() ? new SourcedSearchExpression(
154 searchFields.getGroupId() ) : new UserInputSearchExpression( searchFields.getGroupId() ) ),
155 BooleanClause.Occur.MUST );
158 if ( StringUtils.isNotBlank( searchFields.getArtifactId() ) )
160 q.add( indexer.constructQuery( MAVEN.ARTIFACT_ID,
161 searchFields.isExactSearch()
162 ? new SourcedSearchExpression( searchFields.getArtifactId() )
163 : new UserInputSearchExpression( searchFields.getArtifactId() ) ),
164 BooleanClause.Occur.MUST );
167 if ( StringUtils.isNotBlank( searchFields.getVersion() ) )
169 q.add( indexer.constructQuery( MAVEN.VERSION, searchFields.isExactSearch() ? new SourcedSearchExpression(
170 searchFields.getVersion() ) : new SourcedSearchExpression( searchFields.getVersion() ) ),
171 BooleanClause.Occur.MUST );
174 if ( StringUtils.isNotBlank( searchFields.getPackaging() ) )
176 q.add( indexer.constructQuery( MAVEN.PACKAGING, searchFields.isExactSearch() ? new SourcedSearchExpression(
177 searchFields.getPackaging() ) : new UserInputSearchExpression( searchFields.getPackaging() ) ),
178 BooleanClause.Occur.MUST );
181 if ( StringUtils.isNotBlank( searchFields.getClassName() ) )
183 q.add( indexer.constructQuery( MAVEN.CLASSNAMES,
184 new UserInputSearchExpression( searchFields.getClassName() ) ),
185 BooleanClause.Occur.MUST );
188 if ( StringUtils.isNotBlank( searchFields.getBundleSymbolicName() ) )
190 q.add( indexer.constructQuery( OSGI.SYMBOLIC_NAME,
191 new UserInputSearchExpression( searchFields.getBundleSymbolicName() ) ),
192 BooleanClause.Occur.MUST );
195 if ( StringUtils.isNotBlank( searchFields.getBundleVersion() ) )
197 q.add( indexer.constructQuery( OSGI.VERSION,
198 new UserInputSearchExpression( searchFields.getBundleVersion() ) ),
199 BooleanClause.Occur.MUST );
202 if ( StringUtils.isNotBlank( searchFields.getBundleExportPackage() ) )
204 q.add( indexer.constructQuery( OSGI.EXPORT_PACKAGE,
205 new UserInputSearchExpression( searchFields.getBundleExportPackage() ) ),
209 if ( StringUtils.isNotBlank( searchFields.getBundleExportService() ) )
211 q.add( indexer.constructQuery( OSGI.EXPORT_SERVICE,
212 new UserInputSearchExpression( searchFields.getBundleExportService() ) ),
216 if ( StringUtils.isNotBlank( searchFields.getBundleImportPackage() ) )
218 q.add( indexer.constructQuery( OSGI.IMPORT_PACKAGE,
219 new UserInputSearchExpression( searchFields.getBundleImportPackage() ) ),
223 if ( StringUtils.isNotBlank( searchFields.getBundleName() ) )
225 q.add( indexer.constructQuery( OSGI.NAME, new UserInputSearchExpression( searchFields.getBundleName() ) ),
229 if ( StringUtils.isNotBlank( searchFields.getBundleImportPackage() ) )
231 q.add( indexer.constructQuery( OSGI.IMPORT_PACKAGE,
232 new UserInputSearchExpression( searchFields.getBundleImportPackage() ) ),
236 if ( StringUtils.isNotBlank( searchFields.getBundleRequireBundle() ) )
238 q.add( indexer.constructQuery( OSGI.REQUIRE_BUNDLE,
239 new UserInputSearchExpression( searchFields.getBundleRequireBundle() ) ),
243 if ( StringUtils.isNotBlank( searchFields.getClassifier() ) )
245 q.add( indexer.constructQuery( MAVEN.CLASSIFIER, searchFields.isExactSearch() ? new SourcedSearchExpression(
246 searchFields.getClassifier() ) : new UserInputSearchExpression( searchFields.getClassifier() ) ),
249 else if ( searchFields.isExactSearch() )
251 //TODO improvement in case of exact search and no classifier we must query for classifier with null value
252 // currently it's done in DefaultSearchService with some filtering
255 if ( q.getClauses() == null || q.getClauses().length <= 0 )
257 throw new RepositorySearchException( "No search fields set." );
259 if (q.getClauses()!=null) {
260 log.debug("CLAUSES ", q.getClauses());
261 for (BooleanClause cl : q.getClauses()) {
262 log.debug("Clause ",cl);
266 return search( limits, q, indexingContextIds, Collections.<ArtifactInfoFilter>emptyList(),
267 searchFields.getRepositories(), searchFields.isIncludePomArtifacts() );
270 private static class NullSearch
271 implements SearchTyped, SearchExpression
273 private static final NullSearch INSTANCE = new NullSearch();
276 public String getStringValue()
278 return "[[NULL_VALUE]]";
282 public SearchType getSearchType()
284 return SearchType.EXACT;
288 private SearchResults search( SearchResultLimits limits, BooleanQuery q, List<String> indexingContextIds,
289 List<? extends ArtifactInfoFilter> filters, List<String> selectedRepos,
290 boolean includePoms )
291 throws RepositorySearchException
296 FlatSearchRequest request = new FlatSearchRequest( q );
298 request.setContexts( getIndexingContexts( indexingContextIds ) );
299 if ( limits != null )
301 // we apply limits only when first page asked
302 if ( limits.getSelectedPage() == 0 )
304 request.setCount( limits.getPageSize() * ( Math.max( 1, limits.getSelectedPage() ) ) );
308 FlatSearchResponse response = indexer.searchFlat( request );
310 if ( response == null || response.getTotalHits() == 0 )
312 SearchResults results = new SearchResults();
313 results.setLimits( limits );
317 return convertToSearchResults( response, limits, filters, selectedRepos, includePoms );
319 catch ( IOException e )
321 throw new RepositorySearchException( e.getMessage(), e );
323 catch ( RepositoryAdminException e )
325 throw new RepositorySearchException( e.getMessage(), e );
330 private IndexingContext getIndexingContext(String id) {
332 if (StringUtils.startsWith(id, "remote-")) {
333 repoId = StringUtils.substringAfter(id, "remote-");
337 Repository repo = repositoryRegistry.getRepository(repoId);
341 if (repo.getIndexingContext()!=null) {
343 return repo.getIndexingContext().getBaseContext(IndexingContext.class);
344 } catch (UnsupportedBaseContextException e) {
353 private List<IndexingContext> getIndexingContexts( List<String> ids )
355 List<IndexingContext> contexts = new ArrayList<>( ids.size() );
357 for ( String id : ids )
359 IndexingContext context = getIndexingContext(id);
360 if ( context != null )
362 contexts.add( context );
366 log.warn( "context with id {} not exists", id );
373 private void constructQuery( String term, BooleanQuery q )
375 q.add( indexer.constructQuery( MAVEN.GROUP_ID, new UserInputSearchExpression( term ) ), Occur.SHOULD );
376 q.add( indexer.constructQuery( MAVEN.ARTIFACT_ID, new UserInputSearchExpression( term ) ), Occur.SHOULD );
377 q.add( indexer.constructQuery( MAVEN.VERSION, new UserInputSearchExpression( term ) ), Occur.SHOULD );
378 q.add( indexer.constructQuery( MAVEN.PACKAGING, new UserInputSearchExpression( term ) ), Occur.SHOULD );
379 q.add( indexer.constructQuery( MAVEN.CLASSNAMES, new UserInputSearchExpression( term ) ), Occur.SHOULD );
382 // new WildcardQuery( new Term( MAVEN.CLASSNAMES.getFieldName(), "*" ) );
383 //q.add( query, Occur.MUST_NOT );
384 // olamy IMHO we could set this option as at least one must match
385 //q.setMinimumNumberShouldMatch( 1 );
390 * @param selectedRepos
391 * @return indexing contextId used
393 private List<String> addIndexingContexts( List<String> selectedRepos )
395 Set<String> indexingContextIds = new HashSet<>();
396 for ( String repo : selectedRepos )
400 Repository rRepo = repositoryRegistry.getRepository(repo);
405 if (rRepo.getType().equals(RepositoryType.MAVEN)) {
406 assert rRepo.getIndexingContext() != null;
407 IndexingContext context = rRepo.getIndexingContext().getBaseContext(IndexingContext.class);
408 if (context.isSearchable()) {
409 indexingContextIds.addAll(getRemoteIndexingContextIds(repo));
410 indexingContextIds.add(context.getId());
412 log.warn("indexingContext with id {} not searchable", rRepo.getId());
419 log.warn( "Repository '{}' not found in configuration.", repo );
422 catch ( RepositorySearchException e )
424 log.warn( "RepositorySearchException occured while accessing index of repository '{}' : {}", repo,
427 } catch (UnsupportedBaseContextException e) {
428 log.error("Fatal situation: Maven repository without IndexingContext found.");
433 return new ArrayList<>( indexingContextIds );
438 public Set<String> getRemoteIndexingContextIds( String managedRepoId )
439 throws RepositorySearchException
441 Set<String> ids = new HashSet<>();
443 List<ProxyConnector> proxyConnectors = null;
446 proxyConnectors = proxyConnectorAdmin.getProxyConnectorAsMap().get( managedRepoId );
448 catch ( RepositoryAdminException e )
450 throw new RepositorySearchException( e.getMessage(), e );
453 if ( proxyConnectors == null || proxyConnectors.isEmpty() )
458 for ( ProxyConnector proxyConnector : proxyConnectors )
460 String remoteId = "remote-" + proxyConnector.getTargetRepoId();
461 RemoteRepository repo = repositoryRegistry.getRemoteRepository(proxyConnector.getTargetRepoId());
462 if (repo.getType()==RepositoryType.MAVEN) {
464 IndexingContext context = repo.getIndexingContext() != null ? repo.getIndexingContext().getBaseContext(IndexingContext.class) : null;
465 if (context!=null && context.isSearchable()) {
468 } catch (UnsupportedBaseContextException e) {
478 public Collection<String> getAllGroupIds( String principal, List<String> selectedRepos )
479 throws RepositorySearchException
481 List<IndexingContext> indexContexts = getIndexingContexts( selectedRepos );
483 if ( indexContexts == null || indexContexts.isEmpty() )
485 return Collections.emptyList();
490 Set<String> allGroupIds = new HashSet<>();
491 for ( IndexingContext indexingContext : indexContexts )
493 allGroupIds.addAll( indexingContext.getAllGroups() );
497 catch ( IOException e )
499 throw new RepositorySearchException( e.getMessage(), e );
504 private SearchResults convertToSearchResults( FlatSearchResponse response, SearchResultLimits limits,
505 List<? extends ArtifactInfoFilter> artifactInfoFilters,
506 List<String> selectedRepos, boolean includePoms )
507 throws RepositoryAdminException
509 SearchResults results = new SearchResults();
510 Set<ArtifactInfo> artifactInfos = response.getResults();
512 for ( ArtifactInfo artifactInfo : artifactInfos )
514 if ( StringUtils.equalsIgnoreCase( "pom", artifactInfo.getFileExtension() ) && !includePoms )
518 String id = SearchUtil.getHitId( artifactInfo.getGroupId(), //
519 artifactInfo.getArtifactId(), //
520 artifactInfo.getClassifier(), //
521 artifactInfo.getPackaging() );
522 Map<String, SearchResultHit> hitsMap = results.getHitsMap();
525 if ( !applyArtifactInfoFilters( artifactInfo, artifactInfoFilters, hitsMap ) )
530 SearchResultHit hit = hitsMap.get( id );
533 if ( !hit.getVersions().contains( artifactInfo.getVersion() ) )
535 hit.addVersion( artifactInfo.getVersion() );
540 hit = new SearchResultHit();
541 hit.setArtifactId( artifactInfo.getArtifactId() );
542 hit.setGroupId( artifactInfo.getGroupId() );
543 hit.setRepositoryId( artifactInfo.getRepository() );
544 hit.addVersion( artifactInfo.getVersion() );
545 hit.setBundleExportPackage( artifactInfo.getBundleExportPackage() );
546 hit.setBundleExportService( artifactInfo.getBundleExportService() );
547 hit.setBundleSymbolicName( artifactInfo.getBundleSymbolicName() );
548 hit.setBundleVersion( artifactInfo.getBundleVersion() );
549 hit.setBundleDescription( artifactInfo.getBundleDescription() );
550 hit.setBundleDocUrl( artifactInfo.getBundleDocUrl() );
551 hit.setBundleRequireBundle( artifactInfo.getBundleRequireBundle() );
552 hit.setBundleImportPackage( artifactInfo.getBundleImportPackage() );
553 hit.setBundleLicense( artifactInfo.getBundleLicense() );
554 hit.setBundleName( artifactInfo.getBundleName() );
555 hit.setContext( artifactInfo.getContext() );
556 hit.setGoals( artifactInfo.getGoals() );
557 hit.setPrefix( artifactInfo.getPrefix() );
558 hit.setPackaging( artifactInfo.getPackaging() );
559 hit.setClassifier( artifactInfo.getClassifier() );
560 hit.setFileExtension( artifactInfo.getFileExtension() );
561 hit.setUrl( getBaseUrl( artifactInfo, selectedRepos ) );
564 results.addHit( id, hit );
567 results.setTotalHits( response.getTotalHitsCount() );
568 results.setTotalHitsMapSize( results.getHitsMap().values().size() );
569 results.setReturnedHitsCount( response.getReturnedHitsCount() );
570 results.setLimits( limits );
572 if ( limits == null || limits.getSelectedPage() == SearchResultLimits.ALL_PAGES )
578 return paginate( results );
583 * calculate baseUrl without the context and base Archiva Url
585 * @param artifactInfo
588 protected String getBaseUrl( ArtifactInfo artifactInfo, List<String> selectedRepos )
589 throws RepositoryAdminException
591 StringBuilder sb = new StringBuilder();
592 if ( StringUtils.startsWith( artifactInfo.getContext(), "remote-" ) )
594 // it's a remote index result we search a managed which proxying this remote and on which
595 // current user has read karma
596 String managedRepoId =
597 getManagedRepoId( StringUtils.substringAfter( artifactInfo.getContext(), "remote-" ), selectedRepos );
598 if ( managedRepoId != null )
600 sb.append( '/' ).append( managedRepoId );
601 artifactInfo.setContext( managedRepoId );
606 sb.append( '/' ).append( artifactInfo.getContext() );
609 sb.append( '/' ).append( StringUtils.replaceChars( artifactInfo.getGroupId(), '.', '/' ) );
610 sb.append( '/' ).append( artifactInfo.getArtifactId() );
611 sb.append( '/' ).append( artifactInfo.getVersion() );
612 sb.append( '/' ).append( artifactInfo.getArtifactId() );
613 sb.append( '-' ).append( artifactInfo.getVersion() );
614 if ( StringUtils.isNotBlank( artifactInfo.getClassifier() ) )
616 sb.append( '-' ).append( artifactInfo.getClassifier() );
618 // maven-plugin packaging is a jar
619 if ( StringUtils.equals( "maven-plugin", artifactInfo.getPackaging() ) )
625 sb.append( '.' ).append( artifactInfo.getPackaging() );
628 return sb.toString();
632 * return a managed repo for a remote result
635 * @param selectedRepos
637 * @throws RepositoryAdminException
639 private String getManagedRepoId( String remoteRepo, List<String> selectedRepos )
640 throws RepositoryAdminException
642 Map<String, List<ProxyConnector>> proxyConnectorMap = proxyConnectorAdmin.getProxyConnectorAsMap();
643 if ( proxyConnectorMap == null || proxyConnectorMap.isEmpty() )
647 if ( selectedRepos != null && !selectedRepos.isEmpty() )
649 for ( Map.Entry<String, List<ProxyConnector>> entry : proxyConnectorMap.entrySet() )
651 if ( selectedRepos.contains( entry.getKey() ) )
653 for ( ProxyConnector proxyConnector : entry.getValue() )
655 if ( StringUtils.equals( remoteRepo, proxyConnector.getTargetRepoId() ) )
657 return proxyConnector.getSourceRepoId();
664 // we don't find in search selected repos so return the first one
665 for ( Map.Entry<String, List<ProxyConnector>> entry : proxyConnectorMap.entrySet() )
668 for ( ProxyConnector proxyConnector : entry.getValue() )
670 if ( StringUtils.equals( remoteRepo, proxyConnector.getTargetRepoId() ) )
672 return proxyConnector.getSourceRepoId();
680 private boolean applyArtifactInfoFilters( ArtifactInfo artifactInfo,
681 List<? extends ArtifactInfoFilter> artifactInfoFilters,
682 Map<String, SearchResultHit> currentResult )
684 if ( artifactInfoFilters == null || artifactInfoFilters.isEmpty() )
689 ArchivaArtifactModel artifact = new ArchivaArtifactModel();
690 artifact.setArtifactId( artifactInfo.getArtifactId() );
691 artifact.setClassifier( artifactInfo.getClassifier() );
692 artifact.setGroupId( artifactInfo.getGroupId() );
693 artifact.setRepositoryId( artifactInfo.getRepository() );
694 artifact.setVersion( artifactInfo.getVersion() );
695 artifact.setChecksumMD5( artifactInfo.getMd5() );
696 artifact.setChecksumSHA1( artifactInfo.getSha1() );
697 for ( ArtifactInfoFilter filter : artifactInfoFilters )
699 if ( !filter.addArtifactInResult( artifact, currentResult ) )
707 protected SearchResults paginate( SearchResults results )
709 SearchResultLimits limits = results.getLimits();
710 SearchResults paginated = new SearchResults();
712 // ( limits.getPageSize() * ( Math.max( 1, limits.getSelectedPage() ) ) );
714 int fetchCount = limits.getPageSize();
715 int offset = ( limits.getSelectedPage() * limits.getPageSize() );
717 if ( fetchCount > results.getTotalHits() )
719 fetchCount = results.getTotalHits();
723 if ( offset < results.getTotalHits() )
725 // only process if the offset is within the hit count.
726 for ( int i = 0; i < fetchCount; i++ )
728 // Stop fetching if we are past the total # of available hits.
729 if ( offset + i >= results.getHits().size() )
734 SearchResultHit hit = results.getHits().get( ( offset + i ) );
737 String id = SearchUtil.getHitId( hit.getGroupId(), hit.getArtifactId(), hit.getClassifier(),
738 hit.getPackaging() );
739 paginated.addHit( id, hit );
747 paginated.setTotalHits( results.getTotalHits() );
748 paginated.setReturnedHitsCount( paginated.getHits().size() );
749 paginated.setTotalHitsMapSize( results.getTotalHitsMapSize() );
750 paginated.setLimits( limits );