]> source.dussan.org Git - archiva.git/blob
4b5ad3b369ae84ab8400b17034036f29257b55ac
[archiva.git] /
1 package org.apache.archiva.indexer.search;
2
3 /*
4  * Licensed to the Apache Software Foundation (ASF) under one
5  * or more contributor license agreements.  See the NOTICE file
6  * distributed with this work for additional information
7  * regarding copyright ownership.  The ASF licenses this file
8  * to you under the Apache License, Version 2.0 (the
9  * "License"); you may not use this file except in compliance
10  * with the License.  You may obtain a copy of the License at
11  *
12  *  http://www.apache.org/licenses/LICENSE-2.0
13  *
14  * Unless required by applicable law or agreed to in writing,
15  * software distributed under the License is distributed on an
16  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17  * KIND, either express or implied.  See the License for the
18  * specific language governing permissions and limitations
19  * under the License.
20  */
21
22 import org.apache.archiva.admin.model.RepositoryAdminException;
23 import org.apache.archiva.admin.model.beans.ManagedRepository;
24 import org.apache.archiva.admin.model.beans.ProxyConnector;
25 import org.apache.archiva.admin.model.managed.ManagedRepositoryAdmin;
26 import org.apache.archiva.admin.model.proxyconnector.ProxyConnectorAdmin;
27 import org.apache.archiva.common.plexusbridge.MavenIndexerUtils;
28 import org.apache.archiva.common.plexusbridge.PlexusSisuBridge;
29 import org.apache.archiva.common.plexusbridge.PlexusSisuBridgeException;
30 import org.apache.archiva.indexer.util.SearchUtil;
31 import org.apache.commons.lang.StringUtils;
32 import org.apache.lucene.search.BooleanClause.Occur;
33 import org.apache.lucene.search.BooleanQuery;
34 import org.apache.maven.index.ArtifactInfo;
35 import org.apache.maven.index.FlatSearchRequest;
36 import org.apache.maven.index.FlatSearchResponse;
37 import org.apache.maven.index.MAVEN;
38 import org.apache.maven.index.NexusIndexer;
39 import org.apache.maven.index.OSGI;
40 import org.apache.maven.index.context.IndexCreator;
41 import org.apache.maven.index.context.IndexingContext;
42 import org.apache.maven.index.context.UnsupportedExistingLuceneIndexException;
43 import org.apache.maven.index.expr.StringSearchExpression;
44 import org.slf4j.Logger;
45 import org.slf4j.LoggerFactory;
46 import org.springframework.stereotype.Service;
47
48 import javax.inject.Inject;
49 import java.io.File;
50 import java.io.IOException;
51 import java.util.ArrayList;
52 import java.util.Collection;
53 import java.util.Collections;
54 import java.util.HashSet;
55 import java.util.List;
56 import java.util.Map;
57 import java.util.Set;
58
59 /**
60  * RepositorySearch implementation which uses the Nexus Indexer for searching.
61  */
62 @Service( "nexusSearch" )
63 public class NexusRepositorySearch
64     implements RepositorySearch
65 {
66     private Logger log = LoggerFactory.getLogger(getClass());
67
68     private NexusIndexer indexer;
69
70     private ManagedRepositoryAdmin managedRepositoryAdmin;
71
72     private ProxyConnectorAdmin proxyConnectorAdmin;
73
74     private MavenIndexerUtils mavenIndexerUtils;
75
76     protected NexusRepositorySearch()
77     {
78         // for test purpose
79     }
80
81     @Inject
82     public NexusRepositorySearch(PlexusSisuBridge plexusSisuBridge, ManagedRepositoryAdmin managedRepositoryAdmin,
83                                  MavenIndexerUtils mavenIndexerUtils, ProxyConnectorAdmin proxyConnectorAdmin)
84         throws PlexusSisuBridgeException
85     {
86         this.indexer = plexusSisuBridge.lookup(NexusIndexer.class);
87         this.managedRepositoryAdmin = managedRepositoryAdmin;
88         this.mavenIndexerUtils = mavenIndexerUtils;
89         this.proxyConnectorAdmin = proxyConnectorAdmin;
90     }
91
92     /**
93      * @see RepositorySearch#search(String, List, String, SearchResultLimits, List)
94      */
95     public SearchResults search(String principal, List<String> selectedRepos, String term, SearchResultLimits limits,
96                                 List<String> previousSearchTerms)
97         throws RepositorySearchException
98     {
99         List<String> indexingContextIds = addIndexingContexts( selectedRepos );
100
101         // since upgrade to nexus 2.0.0, query has changed from g:[QUERIED TERM]* to g:*[QUERIED TERM]*
102         //      resulting to more wildcard searches so we need to increase max clause count
103         BooleanQuery.setMaxClauseCount(Integer.MAX_VALUE);
104         BooleanQuery q = new BooleanQuery();
105
106         if ( previousSearchTerms == null || previousSearchTerms.isEmpty() )
107         {
108             constructQuery(term, q);
109         }
110         else
111         {
112             for ( String previousTerm : previousSearchTerms )
113             {
114                 BooleanQuery iQuery = new BooleanQuery();
115                 constructQuery(previousTerm, iQuery);
116
117                 q.add(iQuery, Occur.MUST);
118             }
119
120             BooleanQuery iQuery = new BooleanQuery();
121             constructQuery(term, iQuery);
122             q.add(iQuery, Occur.MUST);
123         }
124
125         // we retun only artifacts without classifier in quick search, olamy cannot find a way to say with this field empty
126         // FIXME  cannot find a way currently to setup this in constructQuery !!!
127         return search(limits, q, indexingContextIds, NoClassifierArtifactInfoFiler.LIST, principal, selectedRepos);
128
129     }
130
131     /**
132      * @see RepositorySearch#search(String, SearchFields, SearchResultLimits)
133      */
134     public SearchResults search(String principal, SearchFields searchFields, SearchResultLimits limits)
135         throws RepositorySearchException
136     {
137         if ( searchFields.getRepositories() == null )
138         {
139             throw new RepositorySearchException("Repositories cannot be null.");
140         }
141
142         List<String> indexingContextIds = addIndexingContexts(searchFields.getRepositories());
143
144         BooleanQuery q = new BooleanQuery();
145         if ( StringUtils.isNotBlank(searchFields.getGroupId()) )
146         {
147             q.add(indexer.constructQuery(MAVEN.GROUP_ID, new StringSearchExpression(searchFields.getGroupId())),
148                   Occur.MUST);
149         }
150
151         if ( StringUtils.isNotBlank(searchFields.getArtifactId()) )
152         {
153             q.add(indexer.constructQuery(MAVEN.ARTIFACT_ID, new StringSearchExpression(searchFields.getArtifactId())),
154                   Occur.MUST);
155         }
156
157         if ( StringUtils.isNotBlank(searchFields.getVersion()) )
158         {
159             q.add(indexer.constructQuery(MAVEN.VERSION, new StringSearchExpression(searchFields.getVersion())),
160                   Occur.MUST);
161         }
162
163         if ( StringUtils.isNotBlank(searchFields.getPackaging()) )
164         {
165             q.add(indexer.constructQuery(MAVEN.PACKAGING, new StringSearchExpression(searchFields.getPackaging())),
166                   Occur.MUST);
167         }
168
169         if ( StringUtils.isNotBlank(searchFields.getClassName()) )
170         {
171             q.add(indexer.constructQuery(MAVEN.CLASSNAMES, new StringSearchExpression(searchFields.getClassName())),
172                   Occur.MUST);
173         }
174
175         if ( StringUtils.isNotBlank(searchFields.getBundleSymbolicName()) )
176         {
177             q.add(indexer.constructQuery(OSGI.SYMBOLIC_NAME,
178                                          new StringSearchExpression(searchFields.getBundleSymbolicName())), Occur.MUST);
179         }
180
181         if ( StringUtils.isNotBlank(searchFields.getBundleVersion()) )
182         {
183             q.add(indexer.constructQuery(OSGI.VERSION, new StringSearchExpression(searchFields.getBundleVersion())),
184                   Occur.MUST);
185         }
186
187         if ( StringUtils.isNotBlank(searchFields.getBundleExportPackage()) )
188         {
189             q.add(indexer.constructQuery(OSGI.EXPORT_PACKAGE,
190                                          new StringSearchExpression(searchFields.getBundleExportPackage())),
191                   Occur.MUST);
192         }
193
194         if ( StringUtils.isNotBlank(searchFields.getBundleExportService()) )
195         {
196             q.add(indexer.constructQuery(OSGI.EXPORT_SERVICE,
197                                          new StringSearchExpression(searchFields.getBundleExportService())),
198                   Occur.MUST);
199         }
200
201         if ( StringUtils.isNotBlank(searchFields.getBundleImportPackage()) )
202         {
203             q.add(indexer.constructQuery(OSGI.IMPORT_PACKAGE,
204                                          new StringSearchExpression(searchFields.getBundleImportPackage())),
205                   Occur.MUST);
206         }
207
208         if ( StringUtils.isNotBlank(searchFields.getBundleName()) )
209         {
210             q.add(indexer.constructQuery(OSGI.NAME, new StringSearchExpression(searchFields.getBundleName())),
211                   Occur.MUST);
212         }
213
214         if ( StringUtils.isNotBlank(searchFields.getClassifier()) )
215         {
216             q.add(indexer.constructQuery(MAVEN.CLASSIFIER, new StringSearchExpression(searchFields.getClassifier())),
217                   Occur.MUST);
218         }
219
220         if ( q.getClauses() == null || q.getClauses().length <= 0 )
221         {
222             throw new RepositorySearchException("No search fields set.");
223         }
224
225         return search(limits, q, indexingContextIds, Collections.<ArtifactInfoFiler>emptyList(), principal,
226                       searchFields.getRepositories());
227     }
228
229     private SearchResults search(SearchResultLimits limits, BooleanQuery q, List<String> indexingContextIds,
230                                  List<? extends ArtifactInfoFiler> filters, String principal,
231                                  List<String> selectedRepos)
232         throws RepositorySearchException
233     {
234
235         try
236         {
237             FlatSearchRequest request = new FlatSearchRequest(q);
238             request.setContexts(getIndexingContexts(indexingContextIds));
239
240             FlatSearchResponse response = indexer.searchFlat(request);
241
242             if ( response == null || response.getTotalHits() == 0 )
243             {
244                 SearchResults results = new SearchResults();
245                 results.setLimits(limits);
246                 return results;
247             }
248
249             return convertToSearchResults(response, limits, filters, principal, selectedRepos);
250         }
251         catch ( IOException e )
252         {
253             throw new RepositorySearchException(e.getMessage(), e);
254         }
255         catch ( RepositoryAdminException e )
256         {
257             throw new RepositorySearchException(e.getMessage(), e);
258         }
259
260     }
261
262     private List<IndexingContext> getIndexingContexts(List<String> ids)
263     {
264         List<IndexingContext> contexts = new ArrayList<IndexingContext>(ids.size());
265
266         for ( String id : ids )
267         {
268             IndexingContext context = indexer.getIndexingContexts().get(id);
269             if ( context != null )
270             {
271                 contexts.add(context);
272             }
273             else
274             {
275                 log.warn("context with id {} not exists", id);
276             }
277         }
278
279         return contexts;
280     }
281
282     private void constructQuery(String term, BooleanQuery q)
283     {
284         q.add(indexer.constructQuery(MAVEN.GROUP_ID, new StringSearchExpression(term)), Occur.SHOULD);
285         q.add(indexer.constructQuery(MAVEN.ARTIFACT_ID, new StringSearchExpression(term)), Occur.SHOULD);
286         q.add(indexer.constructQuery(MAVEN.VERSION, new StringSearchExpression(term)), Occur.SHOULD);
287         q.add(indexer.constructQuery(MAVEN.PACKAGING, new StringSearchExpression(term)), Occur.SHOULD);
288         q.add(indexer.constructQuery(MAVEN.CLASSNAMES, new StringSearchExpression(term)), Occur.SHOULD);
289
290         //Query query =
291         //    new WildcardQuery( new Term( MAVEN.CLASSNAMES.getFieldName(), "*" ) );
292         //q.add( query, Occur.MUST_NOT );
293         // olamy IMHO we could set this option as at least one must match
294         //q.setMinimumNumberShouldMatch( 1 );
295     }
296
297
298     /**
299      * @param selectedRepos
300      * @return indexing contextId used
301      */
302     private List<String> addIndexingContexts(List<String> selectedRepos)
303     {
304         Set<String> indexingContextIds = new HashSet<String>();
305         for ( String repo : selectedRepos )
306         {
307             try
308             {
309                 ManagedRepository repoConfig = managedRepositoryAdmin.getManagedRepository(repo);
310
311                 if ( repoConfig != null )
312                 {
313                     String indexDir = repoConfig.getIndexDirectory();
314                     File indexDirectory = null;
315                     if ( indexDir != null && !"".equals(indexDir) )
316                     {
317                         indexDirectory = new File(repoConfig.getIndexDirectory());
318                     }
319                     else
320                     {
321                         indexDirectory = new File(repoConfig.getLocation(), ".indexer");
322                     }
323
324                     IndexingContext context = indexer.getIndexingContexts().get(repoConfig.getId());
325                     if ( context != null )
326                     {
327                         // alreday here so no need to record it again
328                         log.debug("index with id {} already exists skip adding it", repoConfig.getId());
329                         // set searchable flag
330                         context.setSearchable(repoConfig.isScanned());
331                         indexingContextIds.add(context.getId());
332                         indexingContextIds.addAll(getRemoteIndexingContextIds(repo));
333                         continue;
334                     }
335
336                     context = indexer.addIndexingContext(repoConfig.getId(), repoConfig.getId(),
337                                                          new File(repoConfig.getLocation()), indexDirectory, null, null,
338                                                          getAllIndexCreators());
339                     context.setSearchable(repoConfig.isScanned());
340                     if ( context.isSearchable() )
341                     {
342                         indexingContextIds.addAll(getRemoteIndexingContextIds(repo));
343                         indexingContextIds.add(context.getId());
344                     }
345                     else
346                     {
347                         log.warn("indexingContext with id {} not searchable", repoConfig.getId());
348                     }
349
350                 }
351                 else
352                 {
353                     log.warn("Repository '" + repo + "' not found in configuration.");
354                 }
355             }
356             catch ( UnsupportedExistingLuceneIndexException e )
357             {
358                 log.warn("Error accessing index of repository '" + repo + "' : " + e.getMessage());
359                 continue;
360             }
361             catch ( IOException e )
362             {
363                 log.warn("IO error occured while accessing index of repository '" + repo + "' : " + e.getMessage());
364                 continue;
365             }
366             catch ( RepositoryAdminException e )
367             {
368                 log.warn("RepositoryAdminException occured while accessing index of repository '" + repo + "' : "
369                              + e.getMessage());
370                 continue;
371             }
372         }
373
374         return new ArrayList<String>(indexingContextIds);
375     }
376
377
378     private Set<String> getRemoteIndexingContextIds(String managedRepoId)
379         throws RepositoryAdminException
380     {
381         Set<String> ids = new HashSet<String>();
382
383         List<ProxyConnector> proxyConnectors = proxyConnectorAdmin.getProxyConnectorAsMap().get(managedRepoId);
384
385         if ( proxyConnectors == null || proxyConnectors.isEmpty() )
386         {
387             return ids;
388         }
389
390         for ( ProxyConnector proxyConnector : proxyConnectors )
391         {
392             String remoteId = "remote-" + proxyConnector.getTargetRepoId();
393             IndexingContext context = indexer.getIndexingContexts().get(remoteId);
394             if ( context != null && context.isSearchable() )
395             {
396                 ids.add(remoteId);
397             }
398         }
399
400         return ids;
401     }
402
403     public Collection<String> getAllGroupIds(String principal, List<String> selectedRepos)
404         throws RepositorySearchException
405     {
406         List<IndexingContext> indexContexts = getIndexingContexts( selectedRepos );
407         if (indexContexts == null || indexContexts.isEmpty())
408         {
409             return Collections.emptyList();
410         }
411
412         try
413         {
414             Set<String> allGroupIds = new HashSet<String>(  );
415             for (IndexingContext indexingContext : indexContexts)
416             {
417                 allGroupIds.addAll( indexingContext.getAllGroups() );
418             }
419             return allGroupIds;
420         } catch ( IOException e )
421         {
422             throw new RepositorySearchException( e.getMessage(), e );
423         }
424     }
425
426     protected List<? extends IndexCreator> getAllIndexCreators()
427     {
428         return mavenIndexerUtils.getAllIndexCreators();
429     }
430
431
432     private SearchResults convertToSearchResults(FlatSearchResponse response, SearchResultLimits limits,
433                                                  List<? extends ArtifactInfoFiler> artifactInfoFilers, String principal,
434                                                  List<String> selectedRepos)
435         throws RepositoryAdminException
436     {
437         SearchResults results = new SearchResults();
438         Set<ArtifactInfo> artifactInfos = response.getResults();
439
440         for ( ArtifactInfo artifactInfo : artifactInfos )
441         {
442             String id = SearchUtil.getHitId(artifactInfo.groupId, artifactInfo.artifactId, artifactInfo.classifier,
443                                             artifactInfo.packaging);
444             Map<String, SearchResultHit> hitsMap = results.getHitsMap();
445
446             if ( !applyArtifactInfoFilters(artifactInfo, artifactInfoFilers, hitsMap) )
447             {
448                 continue;
449             }
450
451             SearchResultHit hit = hitsMap.get(id);
452             if ( hit != null )
453             {
454                 if ( !hit.getVersions().contains(artifactInfo.version) )
455                 {
456                     hit.addVersion(artifactInfo.version);
457                 }
458             }
459             else
460             {
461                 hit = new SearchResultHit();
462                 hit.setArtifactId(artifactInfo.artifactId);
463                 hit.setGroupId(artifactInfo.groupId);
464                 hit.setRepositoryId(artifactInfo.repository);
465                 hit.addVersion(artifactInfo.version);
466                 hit.setBundleExportPackage(artifactInfo.bundleExportPackage);
467                 hit.setBundleExportService(artifactInfo.bundleExportService);
468                 hit.setBundleSymbolicName(artifactInfo.bundleSymbolicName);
469                 hit.setBundleVersion(artifactInfo.bundleVersion);
470                 hit.setBundleDescription(artifactInfo.bundleDescription);
471                 hit.setBundleDocUrl(artifactInfo.bundleDocUrl);
472                 hit.setBundleRequireBundle(artifactInfo.bundleRequireBundle);
473                 hit.setBundleImportPackage(artifactInfo.bundleImportPackage);
474                 hit.setBundleLicense(artifactInfo.bundleLicense);
475                 hit.setBundleName(artifactInfo.bundleName);
476                 hit.setContext(artifactInfo.context);
477                 hit.setGoals(artifactInfo.goals);
478                 hit.setPrefix(artifactInfo.prefix);
479                 hit.setPackaging(artifactInfo.packaging);
480                 hit.setClassifier(artifactInfo.classifier);
481                 hit.setUrl(getBaseUrl(artifactInfo, selectedRepos));
482             }
483
484             results.addHit(id, hit);
485         }
486
487         results.setTotalHits(response.getTotalHitsCount());
488         results.setTotalHitsMapSize(results.getHitsMap().values().size());
489         results.setReturnedHitsCount(response.getReturnedHitsCount());
490         results.setLimits(limits);
491
492         if ( limits == null || limits.getSelectedPage() == SearchResultLimits.ALL_PAGES )
493         {
494             return results;
495         }
496         else
497         {
498             return paginate(results);
499         }
500     }
501
502     /**
503      * calculate baseUrl without the context and base Archiva Url
504      *
505      * @param artifactInfo
506      * @return
507      */
508     protected String getBaseUrl(ArtifactInfo artifactInfo, List<String> selectedRepos)
509         throws RepositoryAdminException
510     {
511         StringBuilder sb = new StringBuilder();
512         if ( StringUtils.startsWith(artifactInfo.context, "remote-") )
513         {
514             // it's a remote index result we search a managed which proxying this remote and on which
515             // current user has read karma
516             String managedRepoId =
517                 getManagedRepoId(StringUtils.substringAfter(artifactInfo.context, "remote-"), selectedRepos);
518             if ( managedRepoId != null )
519             {
520                 sb.append('/').append(managedRepoId);
521             }
522         }
523         else
524         {
525             sb.append('/').append(artifactInfo.context);
526         }
527
528         sb.append('/').append(StringUtils.replaceChars(artifactInfo.groupId, '.', '/'));
529         sb.append('/').append(artifactInfo.artifactId);
530         sb.append('/').append(artifactInfo.version);
531         sb.append('/').append(artifactInfo.artifactId);
532         sb.append('-').append(artifactInfo.version);
533         if ( StringUtils.isNotBlank(artifactInfo.classifier) )
534         {
535             sb.append('-').append(artifactInfo.classifier);
536         }
537         // maven-plugin packaging is a jar
538         if ( StringUtils.equals("maven-plugin", artifactInfo.packaging) )
539         {
540             sb.append("jar");
541         }
542         else
543         {
544             sb.append('.').append(artifactInfo.packaging);
545         }
546
547         return sb.toString();
548     }
549
550     /**
551      * return a managed repo for a remote result
552      *
553      * @param remoteRepo
554      * @param selectedRepos
555      * @return
556      * @throws RepositoryAdminException
557      */
558     private String getManagedRepoId(String remoteRepo, List<String> selectedRepos)
559         throws RepositoryAdminException
560     {
561         Map<String, List<ProxyConnector>> proxyConnectorMap = proxyConnectorAdmin.getProxyConnectorAsMap();
562         if ( proxyConnectorMap == null || proxyConnectorMap.isEmpty() )
563         {
564             return null;
565         }
566         if ( selectedRepos != null && !selectedRepos.isEmpty() )
567         {
568             for ( Map.Entry<String, List<ProxyConnector>> entry : proxyConnectorMap.entrySet() )
569             {
570                 if ( selectedRepos.contains(entry.getKey()) )
571                 {
572                     for ( ProxyConnector proxyConnector : entry.getValue() )
573                     {
574                         if ( StringUtils.equals(remoteRepo, proxyConnector.getTargetRepoId()) )
575                         {
576                             return proxyConnector.getSourceRepoId();
577                         }
578                     }
579                 }
580             }
581         }
582
583         // we don't find in search selected repos so return the first one
584         for ( Map.Entry<String, List<ProxyConnector>> entry : proxyConnectorMap.entrySet() )
585         {
586
587             for ( ProxyConnector proxyConnector : entry.getValue() )
588             {
589                 if ( StringUtils.equals(remoteRepo, proxyConnector.getTargetRepoId()) )
590                 {
591                     return proxyConnector.getSourceRepoId();
592                 }
593             }
594
595         }
596         return null;
597     }
598
599     private boolean applyArtifactInfoFilters(ArtifactInfo artifactInfo,
600                                              List<? extends ArtifactInfoFiler> artifactInfoFilers,
601                                              Map<String, SearchResultHit> currentResult)
602     {
603         if ( artifactInfoFilers == null || artifactInfoFilers.isEmpty() )
604         {
605             return true;
606         }
607
608         for ( ArtifactInfoFiler filter : artifactInfoFilers )
609         {
610             if ( !filter.addArtifactInResult(artifactInfo, currentResult) )
611             {
612                 return false;
613             }
614         }
615         return true;
616     }
617
618     protected SearchResults paginate(SearchResults results)
619     {
620         SearchResultLimits limits = results.getLimits();
621         SearchResults paginated = new SearchResults();
622
623         int fetchCount = limits.getPageSize();
624         int offset = ( limits.getSelectedPage() * limits.getPageSize() );
625
626         if ( fetchCount > results.getTotalHits() )
627         {
628             fetchCount = results.getTotalHits();
629         }
630
631         // Goto offset.
632         if ( offset < results.getTotalHits() )
633         {
634             // only process if the offset is within the hit count.
635             for ( int i = 0; i < fetchCount; i++ )
636             {
637                 // Stop fetching if we are past the total # of available hits.
638                 if ( offset + i >= results.getHits().size() )
639                 {
640                     break;
641                 }
642
643                 SearchResultHit hit = results.getHits().get(( offset + i ));
644                 if ( hit != null )
645                 {
646                     String id = SearchUtil.getHitId(hit.getGroupId(), hit.getArtifactId(), hit.getClassifier(),
647                                                     hit.getPackaging());
648                     paginated.addHit(id, hit);
649                 }
650                 else
651                 {
652                     break;
653                 }
654             }
655         }
656         paginated.setTotalHits(results.getTotalHits());
657         paginated.setReturnedHitsCount(paginated.getHits().size());
658         paginated.setTotalHitsMapSize( results.getTotalHitsMapSize() );
659         paginated.setLimits(limits);
660
661         return paginated;
662     }
663 }