]> source.dussan.org Git - archiva.git/blob
a44c09989a2d0ea08db8fbd29bd6edc828df0539
[archiva.git] /
1 package org.apache.archiva.indexer.search;
2
3 /*
4  * Licensed to the Apache Software Foundation (ASF) under one
5  * or more contributor license agreements.  See the NOTICE file
6  * distributed with this work for additional information
7  * regarding copyright ownership.  The ASF licenses this file
8  * to you under the Apache License, Version 2.0 (the
9  * "License"); you may not use this file except in compliance
10  * with the License.  You may obtain a copy of the License at
11  *
12  *  http://www.apache.org/licenses/LICENSE-2.0
13  *
14  * Unless required by applicable law or agreed to in writing,
15  * software distributed under the License is distributed on an
16  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17  * KIND, either express or implied.  See the License for the
18  * specific language governing permissions and limitations
19  * under the License.
20  */
21
22 import org.apache.archiva.admin.model.RepositoryAdminException;
23 import org.apache.archiva.admin.model.beans.ManagedRepository;
24 import org.apache.archiva.admin.model.beans.ProxyConnector;
25 import org.apache.archiva.admin.model.managed.ManagedRepositoryAdmin;
26 import org.apache.archiva.admin.model.proxyconnector.ProxyConnectorAdmin;
27 import org.apache.archiva.common.plexusbridge.MavenIndexerUtils;
28 import org.apache.archiva.common.plexusbridge.PlexusSisuBridge;
29 import org.apache.archiva.common.plexusbridge.PlexusSisuBridgeException;
30 import org.apache.archiva.indexer.util.SearchUtil;
31 import org.apache.commons.lang.StringUtils;
32 import org.apache.lucene.search.BooleanClause.Occur;
33 import org.apache.lucene.search.BooleanQuery;
34 import org.apache.maven.index.ArtifactInfo;
35 import org.apache.maven.index.FlatSearchRequest;
36 import org.apache.maven.index.FlatSearchResponse;
37 import org.apache.maven.index.MAVEN;
38 import org.apache.maven.index.NexusIndexer;
39 import org.apache.maven.index.OSGI;
40 import org.apache.maven.index.context.IndexCreator;
41 import org.apache.maven.index.context.IndexingContext;
42 import org.apache.maven.index.context.UnsupportedExistingLuceneIndexException;
43 import org.apache.maven.index.expr.StringSearchExpression;
44 import org.slf4j.Logger;
45 import org.slf4j.LoggerFactory;
46 import org.springframework.stereotype.Service;
47
48 import javax.inject.Inject;
49 import java.io.File;
50 import java.io.IOException;
51 import java.util.ArrayList;
52 import java.util.Collections;
53 import java.util.HashSet;
54 import java.util.List;
55 import java.util.Map;
56 import java.util.Set;
57
58 /**
59  * RepositorySearch implementation which uses the Nexus Indexer for searching.
60  */
61 @Service( "nexusSearch" )
62 public class NexusRepositorySearch
63     implements RepositorySearch
64 {
65     private Logger log = LoggerFactory.getLogger(getClass());
66
67     private NexusIndexer indexer;
68
69     private ManagedRepositoryAdmin managedRepositoryAdmin;
70
71     private ProxyConnectorAdmin proxyConnectorAdmin;
72
73     private MavenIndexerUtils mavenIndexerUtils;
74
75     protected NexusRepositorySearch()
76     {
77         // for test purpose
78     }
79
80     @Inject
81     public NexusRepositorySearch(PlexusSisuBridge plexusSisuBridge, ManagedRepositoryAdmin managedRepositoryAdmin,
82                                  MavenIndexerUtils mavenIndexerUtils, ProxyConnectorAdmin proxyConnectorAdmin)
83         throws PlexusSisuBridgeException
84     {
85         this.indexer = plexusSisuBridge.lookup(NexusIndexer.class);
86         this.managedRepositoryAdmin = managedRepositoryAdmin;
87         this.mavenIndexerUtils = mavenIndexerUtils;
88         this.proxyConnectorAdmin = proxyConnectorAdmin;
89     }
90
91     /**
92      * @see RepositorySearch#search(String, List, String, SearchResultLimits, List)
93      */
94     public SearchResults search(String principal, List<String> selectedRepos, String term, SearchResultLimits limits,
95                                 List<String> previousSearchTerms)
96         throws RepositorySearchException
97     {
98         List<String> indexingContextIds = addIndexingContexts(selectedRepos);
99
100         // since upgrade to nexus 2.0.0, query has changed from g:[QUERIED TERM]* to g:*[QUERIED TERM]*
101         //      resulting to more wildcard searches so we need to increase max clause count
102         BooleanQuery.setMaxClauseCount(Integer.MAX_VALUE);
103         BooleanQuery q = new BooleanQuery();
104
105         if ( previousSearchTerms == null || previousSearchTerms.isEmpty() )
106         {
107             constructQuery(term, q);
108         }
109         else
110         {
111             for ( String previousTerm : previousSearchTerms )
112             {
113                 BooleanQuery iQuery = new BooleanQuery();
114                 constructQuery(previousTerm, iQuery);
115
116                 q.add(iQuery, Occur.MUST);
117             }
118
119             BooleanQuery iQuery = new BooleanQuery();
120             constructQuery(term, iQuery);
121             q.add(iQuery, Occur.MUST);
122         }
123
124         // we retun only artifacts without classifier in quick search, olamy cannot find a way to say with this field empty
125         // FIXME  cannot find a way currently to setup this in constructQuery !!!
126         return search(limits, q, indexingContextIds, NoClassifierArtifactInfoFiler.LIST, principal, selectedRepos);
127
128     }
129
130     /**
131      * @see RepositorySearch#search(String, SearchFields, SearchResultLimits)
132      */
133     public SearchResults search(String principal, SearchFields searchFields, SearchResultLimits limits)
134         throws RepositorySearchException
135     {
136         if ( searchFields.getRepositories() == null )
137         {
138             throw new RepositorySearchException("Repositories cannot be null.");
139         }
140
141         List<String> indexingContextIds = addIndexingContexts(searchFields.getRepositories());
142
143         BooleanQuery q = new BooleanQuery();
144         if ( StringUtils.isNotBlank(searchFields.getGroupId()) )
145         {
146             q.add(indexer.constructQuery(MAVEN.GROUP_ID, new StringSearchExpression(searchFields.getGroupId())),
147                   Occur.MUST);
148         }
149
150         if ( StringUtils.isNotBlank(searchFields.getArtifactId()) )
151         {
152             q.add(indexer.constructQuery(MAVEN.ARTIFACT_ID, new StringSearchExpression(searchFields.getArtifactId())),
153                   Occur.MUST);
154         }
155
156         if ( StringUtils.isNotBlank(searchFields.getVersion()) )
157         {
158             q.add(indexer.constructQuery(MAVEN.VERSION, new StringSearchExpression(searchFields.getVersion())),
159                   Occur.MUST);
160         }
161
162         if ( StringUtils.isNotBlank(searchFields.getPackaging()) )
163         {
164             q.add(indexer.constructQuery(MAVEN.PACKAGING, new StringSearchExpression(searchFields.getPackaging())),
165                   Occur.MUST);
166         }
167
168         if ( StringUtils.isNotBlank(searchFields.getClassName()) )
169         {
170             q.add(indexer.constructQuery(MAVEN.CLASSNAMES, new StringSearchExpression(searchFields.getClassName())),
171                   Occur.MUST);
172         }
173
174         if ( StringUtils.isNotBlank(searchFields.getBundleSymbolicName()) )
175         {
176             q.add(indexer.constructQuery(OSGI.SYMBOLIC_NAME,
177                                          new StringSearchExpression(searchFields.getBundleSymbolicName())), Occur.MUST);
178         }
179
180         if ( StringUtils.isNotBlank(searchFields.getBundleVersion()) )
181         {
182             q.add(indexer.constructQuery(OSGI.VERSION, new StringSearchExpression(searchFields.getBundleVersion())),
183                   Occur.MUST);
184         }
185
186         if ( StringUtils.isNotBlank(searchFields.getBundleExportPackage()) )
187         {
188             q.add(indexer.constructQuery(OSGI.EXPORT_PACKAGE,
189                                          new StringSearchExpression(searchFields.getBundleExportPackage())),
190                   Occur.MUST);
191         }
192
193         if ( StringUtils.isNotBlank(searchFields.getBundleExportService()) )
194         {
195             q.add(indexer.constructQuery(OSGI.EXPORT_SERVICE,
196                                          new StringSearchExpression(searchFields.getBundleExportService())),
197                   Occur.MUST);
198         }
199
200         if ( StringUtils.isNotBlank(searchFields.getBundleImportPackage()) )
201         {
202             q.add(indexer.constructQuery(OSGI.IMPORT_PACKAGE,
203                                          new StringSearchExpression(searchFields.getBundleImportPackage())),
204                   Occur.MUST);
205         }
206
207         if ( StringUtils.isNotBlank(searchFields.getBundleName()) )
208         {
209             q.add(indexer.constructQuery(OSGI.NAME, new StringSearchExpression(searchFields.getBundleName())),
210                   Occur.MUST);
211         }
212
213         if ( StringUtils.isNotBlank(searchFields.getClassifier()) )
214         {
215             q.add(indexer.constructQuery(MAVEN.CLASSIFIER, new StringSearchExpression(searchFields.getClassifier())),
216                   Occur.MUST);
217         }
218
219         if ( q.getClauses() == null || q.getClauses().length <= 0 )
220         {
221             throw new RepositorySearchException("No search fields set.");
222         }
223
224         return search(limits, q, indexingContextIds, Collections.<ArtifactInfoFiler>emptyList(), principal,
225                       searchFields.getRepositories());
226     }
227
228     private SearchResults search(SearchResultLimits limits, BooleanQuery q, List<String> indexingContextIds,
229                                  List<? extends ArtifactInfoFiler> filters, String principal,
230                                  List<String> selectedRepos)
231         throws RepositorySearchException
232     {
233
234         try
235         {
236             FlatSearchRequest request = new FlatSearchRequest(q);
237             request.setContexts(getIndexingContexts(indexingContextIds));
238
239             FlatSearchResponse response = indexer.searchFlat(request);
240
241             if ( response == null || response.getTotalHits() == 0 )
242             {
243                 SearchResults results = new SearchResults();
244                 results.setLimits(limits);
245                 return results;
246             }
247
248             return convertToSearchResults(response, limits, filters, principal, selectedRepos);
249         }
250         catch ( IOException e )
251         {
252             throw new RepositorySearchException(e.getMessage(), e);
253         }
254         catch ( RepositoryAdminException e )
255         {
256             throw new RepositorySearchException(e.getMessage(), e);
257         }
258
259     }
260
261     private List<IndexingContext> getIndexingContexts(List<String> ids)
262     {
263         List<IndexingContext> contexts = new ArrayList<IndexingContext>(ids.size());
264
265         for ( String id : ids )
266         {
267             IndexingContext context = indexer.getIndexingContexts().get(id);
268             if ( context != null )
269             {
270                 contexts.add(context);
271             }
272             else
273             {
274                 log.warn("context with id {} not exists", id);
275             }
276         }
277
278         return contexts;
279     }
280
281     private void constructQuery(String term, BooleanQuery q)
282     {
283         q.add(indexer.constructQuery(MAVEN.GROUP_ID, new StringSearchExpression(term)), Occur.SHOULD);
284         q.add(indexer.constructQuery(MAVEN.ARTIFACT_ID, new StringSearchExpression(term)), Occur.SHOULD);
285         q.add(indexer.constructQuery(MAVEN.VERSION, new StringSearchExpression(term)), Occur.SHOULD);
286         q.add(indexer.constructQuery(MAVEN.PACKAGING, new StringSearchExpression(term)), Occur.SHOULD);
287         q.add(indexer.constructQuery(MAVEN.CLASSNAMES, new StringSearchExpression(term)), Occur.SHOULD);
288
289         //Query query =
290         //    new WildcardQuery( new Term( MAVEN.CLASSNAMES.getFieldName(), "*" ) );
291         //q.add( query, Occur.MUST_NOT );
292         // olamy IMHO we could set this option as at least one must match
293         //q.setMinimumNumberShouldMatch( 1 );
294     }
295
296
297     /**
298      * @param selectedRepos
299      * @return indexing contextId used
300      */
301     private List<String> addIndexingContexts(List<String> selectedRepos)
302     {
303         Set<String> indexingContextIds = new HashSet<String>();
304         for ( String repo : selectedRepos )
305         {
306             try
307             {
308                 ManagedRepository repoConfig = managedRepositoryAdmin.getManagedRepository(repo);
309
310                 if ( repoConfig != null )
311                 {
312                     String indexDir = repoConfig.getIndexDirectory();
313                     File indexDirectory = null;
314                     if ( indexDir != null && !"".equals(indexDir) )
315                     {
316                         indexDirectory = new File(repoConfig.getIndexDirectory());
317                     }
318                     else
319                     {
320                         indexDirectory = new File(repoConfig.getLocation(), ".indexer");
321                     }
322
323                     IndexingContext context = indexer.getIndexingContexts().get(repoConfig.getId());
324                     if ( context != null )
325                     {
326                         // alreday here so no need to record it again
327                         log.debug("index with id {} already exists skip adding it", repoConfig.getId());
328                         // set searchable flag
329                         context.setSearchable(repoConfig.isScanned());
330                         indexingContextIds.add(context.getId());
331                         indexingContextIds.addAll(getRemoteIndexingContextIds(repo));
332                         continue;
333                     }
334
335                     context = indexer.addIndexingContext(repoConfig.getId(), repoConfig.getId(),
336                                                          new File(repoConfig.getLocation()), indexDirectory, null, null,
337                                                          getAllIndexCreators());
338                     context.setSearchable(repoConfig.isScanned());
339                     if ( context.isSearchable() )
340                     {
341                         indexingContextIds.addAll(getRemoteIndexingContextIds(repo));
342                         indexingContextIds.add(context.getId());
343                     }
344                     else
345                     {
346                         log.warn("indexingContext with id {} not searchable", repoConfig.getId());
347                     }
348
349                 }
350                 else
351                 {
352                     log.warn("Repository '" + repo + "' not found in configuration.");
353                 }
354             }
355             catch ( UnsupportedExistingLuceneIndexException e )
356             {
357                 log.warn("Error accessing index of repository '" + repo + "' : " + e.getMessage());
358                 continue;
359             }
360             catch ( IOException e )
361             {
362                 log.warn("IO error occured while accessing index of repository '" + repo + "' : " + e.getMessage());
363                 continue;
364             }
365             catch ( RepositoryAdminException e )
366             {
367                 log.warn("RepositoryAdminException occured while accessing index of repository '" + repo + "' : "
368                              + e.getMessage());
369                 continue;
370             }
371         }
372
373         return new ArrayList<String>(indexingContextIds);
374     }
375
376
377     private Set<String> getRemoteIndexingContextIds(String managedRepoId)
378         throws RepositoryAdminException
379     {
380         Set<String> ids = new HashSet<String>();
381
382         List<ProxyConnector> proxyConnectors = proxyConnectorAdmin.getProxyConnectorAsMap().get(managedRepoId);
383
384         if ( proxyConnectors == null || proxyConnectors.isEmpty() )
385         {
386             return ids;
387         }
388
389         for ( ProxyConnector proxyConnector : proxyConnectors )
390         {
391             String remoteId = "remote-" + proxyConnector.getTargetRepoId();
392             IndexingContext context = indexer.getIndexingContexts().get(remoteId);
393             if ( context != null && context.isSearchable() )
394             {
395                 ids.add(remoteId);
396             }
397         }
398
399         return ids;
400     }
401
402
403     protected List<? extends IndexCreator> getAllIndexCreators()
404     {
405         return mavenIndexerUtils.getAllIndexCreators();
406     }
407
408
409     private SearchResults convertToSearchResults(FlatSearchResponse response, SearchResultLimits limits,
410                                                  List<? extends ArtifactInfoFiler> artifactInfoFilers, String principal,
411                                                  List<String> selectedRepos)
412         throws RepositoryAdminException
413     {
414         SearchResults results = new SearchResults();
415         Set<ArtifactInfo> artifactInfos = response.getResults();
416
417         for ( ArtifactInfo artifactInfo : artifactInfos )
418         {
419             String id = SearchUtil.getHitId(artifactInfo.groupId, artifactInfo.artifactId, artifactInfo.classifier,
420                                             artifactInfo.packaging);
421             Map<String, SearchResultHit> hitsMap = results.getHitsMap();
422
423             if ( !applyArtifactInfoFilters(artifactInfo, artifactInfoFilers, hitsMap) )
424             {
425                 continue;
426             }
427
428             SearchResultHit hit = hitsMap.get(id);
429             if ( hit != null )
430             {
431                 if ( !hit.getVersions().contains(artifactInfo.version) )
432                 {
433                     hit.addVersion(artifactInfo.version);
434                 }
435             }
436             else
437             {
438                 hit = new SearchResultHit();
439                 hit.setArtifactId(artifactInfo.artifactId);
440                 hit.setGroupId(artifactInfo.groupId);
441                 hit.setRepositoryId(artifactInfo.repository);
442                 hit.addVersion(artifactInfo.version);
443                 hit.setBundleExportPackage(artifactInfo.bundleExportPackage);
444                 hit.setBundleExportService(artifactInfo.bundleExportService);
445                 hit.setBundleSymbolicName(artifactInfo.bundleSymbolicName);
446                 hit.setBundleVersion(artifactInfo.bundleVersion);
447                 hit.setBundleDescription(artifactInfo.bundleDescription);
448                 hit.setBundleDocUrl(artifactInfo.bundleDocUrl);
449                 hit.setBundleRequireBundle(artifactInfo.bundleRequireBundle);
450                 hit.setBundleImportPackage(artifactInfo.bundleImportPackage);
451                 hit.setBundleLicense(artifactInfo.bundleLicense);
452                 hit.setBundleName(artifactInfo.bundleName);
453                 hit.setContext(artifactInfo.context);
454                 hit.setGoals(artifactInfo.goals);
455                 hit.setPrefix(artifactInfo.prefix);
456                 hit.setPackaging(artifactInfo.packaging);
457                 hit.setClassifier(artifactInfo.classifier);
458                 hit.setUrl(getBaseUrl(artifactInfo, selectedRepos));
459             }
460
461             results.addHit(id, hit);
462         }
463
464         results.setTotalHits(response.getTotalHitsCount());
465         results.setTotalHitsMapSize(results.getHitsMap().values().size());
466         results.setReturnedHitsCount(response.getReturnedHitsCount());
467         results.setLimits(limits);
468
469         if ( limits == null || limits.getSelectedPage() == SearchResultLimits.ALL_PAGES )
470         {
471             return results;
472         }
473         else
474         {
475             return paginate(results);
476         }
477     }
478
479     /**
480      * calculate baseUrl without the context and base Archiva Url
481      *
482      * @param artifactInfo
483      * @return
484      */
485     protected String getBaseUrl(ArtifactInfo artifactInfo, List<String> selectedRepos)
486         throws RepositoryAdminException
487     {
488         StringBuilder sb = new StringBuilder();
489         if ( StringUtils.startsWith(artifactInfo.context, "remote-") )
490         {
491             // it's a remote index result we search a managed which proxying this remote and on which
492             // current user has read karma
493             String managedRepoId =
494                 getManagedRepoId(StringUtils.substringAfter(artifactInfo.context, "remote-"), selectedRepos);
495             if ( managedRepoId != null )
496             {
497                 sb.append('/').append(managedRepoId);
498             }
499         }
500         else
501         {
502             sb.append('/').append(artifactInfo.context);
503         }
504
505         sb.append('/').append(StringUtils.replaceChars(artifactInfo.groupId, '.', '/'));
506         sb.append('/').append(artifactInfo.artifactId);
507         sb.append('/').append(artifactInfo.version);
508         sb.append('/').append(artifactInfo.artifactId);
509         sb.append('-').append(artifactInfo.version);
510         if ( StringUtils.isNotBlank(artifactInfo.classifier) )
511         {
512             sb.append('-').append(artifactInfo.classifier);
513         }
514         // maven-plugin packaging is a jar
515         if ( StringUtils.equals("maven-plugin", artifactInfo.packaging) )
516         {
517             sb.append("jar");
518         }
519         else
520         {
521             sb.append('.').append(artifactInfo.packaging);
522         }
523
524         return sb.toString();
525     }
526
527     /**
528      * return a managed repo for a remote result
529      *
530      * @param remoteRepo
531      * @param selectedRepos
532      * @return
533      * @throws RepositoryAdminException
534      */
535     private String getManagedRepoId(String remoteRepo, List<String> selectedRepos)
536         throws RepositoryAdminException
537     {
538         Map<String, List<ProxyConnector>> proxyConnectorMap = proxyConnectorAdmin.getProxyConnectorAsMap();
539         if ( proxyConnectorMap == null || proxyConnectorMap.isEmpty() )
540         {
541             return null;
542         }
543         if ( selectedRepos != null && !selectedRepos.isEmpty() )
544         {
545             for ( Map.Entry<String, List<ProxyConnector>> entry : proxyConnectorMap.entrySet() )
546             {
547                 if ( selectedRepos.contains(entry.getKey()) )
548                 {
549                     for ( ProxyConnector proxyConnector : entry.getValue() )
550                     {
551                         if ( StringUtils.equals(remoteRepo, proxyConnector.getTargetRepoId()) )
552                         {
553                             return proxyConnector.getSourceRepoId();
554                         }
555                     }
556                 }
557             }
558         }
559
560         // we don't find in search selected repos so return the first one
561         for ( Map.Entry<String, List<ProxyConnector>> entry : proxyConnectorMap.entrySet() )
562         {
563
564             for ( ProxyConnector proxyConnector : entry.getValue() )
565             {
566                 if ( StringUtils.equals(remoteRepo, proxyConnector.getTargetRepoId()) )
567                 {
568                     return proxyConnector.getSourceRepoId();
569                 }
570             }
571
572         }
573         return null;
574     }
575
576     private boolean applyArtifactInfoFilters(ArtifactInfo artifactInfo,
577                                              List<? extends ArtifactInfoFiler> artifactInfoFilers,
578                                              Map<String, SearchResultHit> currentResult)
579     {
580         if ( artifactInfoFilers == null || artifactInfoFilers.isEmpty() )
581         {
582             return true;
583         }
584
585         for ( ArtifactInfoFiler filter : artifactInfoFilers )
586         {
587             if ( !filter.addArtifactInResult(artifactInfo, currentResult) )
588             {
589                 return false;
590             }
591         }
592         return true;
593     }
594
595     protected SearchResults paginate(SearchResults results)
596     {
597         SearchResultLimits limits = results.getLimits();
598         SearchResults paginated = new SearchResults();
599
600         int fetchCount = limits.getPageSize();
601         int offset = ( limits.getSelectedPage() * limits.getPageSize() );
602
603         if ( fetchCount > results.getTotalHits() )
604         {
605             fetchCount = results.getTotalHits();
606         }
607
608         // Goto offset.
609         if ( offset < results.getTotalHits() )
610         {
611             // only process if the offset is within the hit count.
612             for ( int i = 0; i < fetchCount; i++ )
613             {
614                 // Stop fetching if we are past the total # of available hits.
615                 if ( offset + i >= results.getHits().size() )
616                 {
617                     break;
618                 }
619
620                 SearchResultHit hit = results.getHits().get(( offset + i ));
621                 if ( hit != null )
622                 {
623                     String id = SearchUtil.getHitId(hit.getGroupId(), hit.getArtifactId(), hit.getClassifier(),
624                                                     hit.getPackaging());
625                     paginated.addHit(id, hit);
626                 }
627                 else
628                 {
629                     break;
630                 }
631             }
632         }
633         paginated.setTotalHits(results.getTotalHits());
634         paginated.setReturnedHitsCount(paginated.getHits().size());
635         paginated.setTotalHitsMapSize( results.getTotalHitsMapSize() );
636         paginated.setLimits(limits);
637
638         return paginated;
639     }
640 }