]> source.dussan.org Git - archiva.git/blob
d2268546b1aa70ee7e90bf28f3a329d1933840df
[archiva.git] /
1 package org.apache.archiva.indexer.search;
2
3 /*
4  * Licensed to the Apache Software Foundation (ASF) under one
5  * or more contributor license agreements.  See the NOTICE file
6  * distributed with this work for additional information
7  * regarding copyright ownership.  The ASF licenses this file
8  * to you under the Apache License, Version 2.0 (the
9  * "License"); you may not use this file except in compliance
10  * with the License.  You may obtain a copy of the License at
11  *
12  *  http://www.apache.org/licenses/LICENSE-2.0
13  *
14  * Unless required by applicable law or agreed to in writing,
15  * software distributed under the License is distributed on an
16  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17  * KIND, either express or implied.  See the License for the
18  * specific language governing permissions and limitations
19  * under the License.
20  */
21
22 import org.apache.archiva.admin.model.RepositoryAdminException;
23 import org.apache.archiva.admin.model.beans.ManagedRepository;
24 import org.apache.archiva.admin.model.beans.ProxyConnector;
25 import org.apache.archiva.admin.model.managed.ManagedRepositoryAdmin;
26 import org.apache.archiva.admin.model.proxyconnector.ProxyConnectorAdmin;
27 import org.apache.archiva.common.plexusbridge.MavenIndexerUtils;
28 import org.apache.archiva.common.plexusbridge.PlexusSisuBridge;
29 import org.apache.archiva.common.plexusbridge.PlexusSisuBridgeException;
30 import org.apache.archiva.indexer.util.SearchUtil;
31 import org.apache.commons.lang.StringUtils;
32 import org.apache.lucene.search.BooleanClause.Occur;
33 import org.apache.lucene.search.BooleanQuery;
34 import org.apache.maven.index.ArtifactInfo;
35 import org.apache.maven.index.FlatSearchRequest;
36 import org.apache.maven.index.FlatSearchResponse;
37 import org.apache.maven.index.MAVEN;
38 import org.apache.maven.index.NexusIndexer;
39 import org.apache.maven.index.OSGI;
40 import org.apache.maven.index.context.IndexCreator;
41 import org.apache.maven.index.context.IndexingContext;
42 import org.apache.maven.index.context.UnsupportedExistingLuceneIndexException;
43 import org.apache.maven.index.expr.StringSearchExpression;
44 import org.slf4j.Logger;
45 import org.slf4j.LoggerFactory;
46 import org.springframework.stereotype.Service;
47
48 import javax.inject.Inject;
49 import java.io.File;
50 import java.io.IOException;
51 import java.util.ArrayList;
52 import java.util.Collections;
53 import java.util.HashSet;
54 import java.util.List;
55 import java.util.Map;
56 import java.util.Set;
57
58 /**
59  * RepositorySearch implementation which uses the Nexus Indexer for searching.
60  */
61 @Service( "nexusSearch" )
62 public class NexusRepositorySearch
63     implements RepositorySearch
64 {
65     private Logger log = LoggerFactory.getLogger(getClass());
66
67     private NexusIndexer indexer;
68
69     private ManagedRepositoryAdmin managedRepositoryAdmin;
70
71     private ProxyConnectorAdmin proxyConnectorAdmin;
72
73     private MavenIndexerUtils mavenIndexerUtils;
74
75     protected NexusRepositorySearch()
76     {
77         // for test purpose
78     }
79
80     @Inject
81     public NexusRepositorySearch(PlexusSisuBridge plexusSisuBridge, ManagedRepositoryAdmin managedRepositoryAdmin,
82                                  MavenIndexerUtils mavenIndexerUtils, ProxyConnectorAdmin proxyConnectorAdmin)
83         throws PlexusSisuBridgeException
84     {
85         this.indexer = plexusSisuBridge.lookup(NexusIndexer.class);
86         this.managedRepositoryAdmin = managedRepositoryAdmin;
87         this.mavenIndexerUtils = mavenIndexerUtils;
88         this.proxyConnectorAdmin = proxyConnectorAdmin;
89     }
90
91     /**
92      * @see RepositorySearch#search(String, List, String, SearchResultLimits, List)
93      */
94     public SearchResults search(String principal, List<String> selectedRepos, String term, SearchResultLimits limits,
95                                 List<String> previousSearchTerms)
96         throws RepositorySearchException
97     {
98         List<String> indexingContextIds = addIndexingContexts(selectedRepos);
99
100         // since upgrade to nexus 2.0.0, query has changed from g:[QUERIED TERM]* to g:*[QUERIED TERM]*
101         //      resulting to more wildcard searches so we need to increase max clause count
102         BooleanQuery.setMaxClauseCount(Integer.MAX_VALUE);
103         BooleanQuery q = new BooleanQuery();
104
105         if ( previousSearchTerms == null || previousSearchTerms.isEmpty() )
106         {
107             constructQuery(term, q);
108         }
109         else
110         {
111             for ( String previousTerm : previousSearchTerms )
112             {
113                 BooleanQuery iQuery = new BooleanQuery();
114                 constructQuery(previousTerm, iQuery);
115
116                 q.add(iQuery, Occur.MUST);
117             }
118
119             BooleanQuery iQuery = new BooleanQuery();
120             constructQuery(term, iQuery);
121             q.add(iQuery, Occur.MUST);
122         }
123
124         // we retun only artifacts without classifier in quick search, olamy cannot find a way to say with this field empty
125         // FIXME  cannot find a way currently to setup this in constructQuery !!!
126         return search(limits, q, indexingContextIds, NoClassifierArtifactInfoFiler.LIST, principal, selectedRepos);
127
128     }
129
130     /**
131      * @see RepositorySearch#search(String, SearchFields, SearchResultLimits)
132      */
133     public SearchResults search(String principal, SearchFields searchFields, SearchResultLimits limits)
134         throws RepositorySearchException
135     {
136         if ( searchFields.getRepositories() == null )
137         {
138             throw new RepositorySearchException("Repositories cannot be null.");
139         }
140
141         List<String> indexingContextIds = addIndexingContexts(searchFields.getRepositories());
142
143         BooleanQuery q = new BooleanQuery();
144         if ( StringUtils.isNotBlank(searchFields.getGroupId()) )
145         {
146             q.add(indexer.constructQuery(MAVEN.GROUP_ID, new StringSearchExpression(searchFields.getGroupId())),
147                   Occur.MUST);
148         }
149
150         if ( StringUtils.isNotBlank(searchFields.getArtifactId()) )
151         {
152             q.add(indexer.constructQuery(MAVEN.ARTIFACT_ID, new StringSearchExpression(searchFields.getArtifactId())),
153                   Occur.MUST);
154         }
155
156         if ( StringUtils.isNotBlank(searchFields.getVersion()) )
157         {
158             q.add(indexer.constructQuery(MAVEN.VERSION, new StringSearchExpression(searchFields.getVersion())),
159                   Occur.MUST);
160         }
161
162         if ( StringUtils.isNotBlank(searchFields.getPackaging()) )
163         {
164             q.add(indexer.constructQuery(MAVEN.PACKAGING, new StringSearchExpression(searchFields.getPackaging())),
165                   Occur.MUST);
166         }
167
168         if ( StringUtils.isNotBlank(searchFields.getClassName()) )
169         {
170             q.add(indexer.constructQuery(MAVEN.CLASSNAMES, new StringSearchExpression(searchFields.getClassName())),
171                   Occur.MUST);
172         }
173
174         if ( StringUtils.isNotBlank(searchFields.getBundleSymbolicName()) )
175         {
176             q.add(indexer.constructQuery(OSGI.SYMBOLIC_NAME,
177                                          new StringSearchExpression(searchFields.getBundleSymbolicName())), Occur.MUST);
178         }
179
180         if ( StringUtils.isNotBlank(searchFields.getBundleVersion()) )
181         {
182             q.add(indexer.constructQuery(OSGI.VERSION, new StringSearchExpression(searchFields.getBundleVersion())),
183                   Occur.MUST);
184         }
185
186         if ( StringUtils.isNotBlank(searchFields.getBundleExportPackage()) )
187         {
188             q.add(indexer.constructQuery(OSGI.EXPORT_PACKAGE,
189                                          new StringSearchExpression(searchFields.getBundleExportPackage())),
190                   Occur.MUST);
191         }
192
193         if ( StringUtils.isNotBlank(searchFields.getBundleExportService()) )
194         {
195             q.add(indexer.constructQuery(OSGI.EXPORT_SERVICE,
196                                          new StringSearchExpression(searchFields.getBundleExportService())),
197                   Occur.MUST);
198         }
199
200         if ( StringUtils.isNotBlank(searchFields.getBundleImportPackage()) )
201         {
202             q.add(indexer.constructQuery(OSGI.IMPORT_PACKAGE,
203                                          new StringSearchExpression(searchFields.getBundleImportPackage())),
204                   Occur.MUST);
205         }
206
207         if ( StringUtils.isNotBlank(searchFields.getBundleName()) )
208         {
209             q.add(indexer.constructQuery(OSGI.NAME, new StringSearchExpression(searchFields.getBundleName())),
210                   Occur.MUST);
211         }
212
213         if ( StringUtils.isNotBlank(searchFields.getClassifier()) )
214         {
215             q.add(indexer.constructQuery(MAVEN.CLASSIFIER, new StringSearchExpression(searchFields.getClassifier())),
216                   Occur.MUST);
217         }
218
219         if ( q.getClauses() == null || q.getClauses().length <= 0 )
220         {
221             throw new RepositorySearchException("No search fields set.");
222         }
223
224         return search(limits, q, indexingContextIds, Collections.<ArtifactInfoFiler>emptyList(), principal,
225                       searchFields.getRepositories());
226     }
227
228     private SearchResults search(SearchResultLimits limits, BooleanQuery q, List<String> indexingContextIds,
229                                  List<? extends ArtifactInfoFiler> filters, String principal,
230                                  List<String> selectedRepos)
231         throws RepositorySearchException
232     {
233
234         try
235         {
236             FlatSearchRequest request = new FlatSearchRequest(q);
237             request.setContexts(getIndexingContexts(indexingContextIds));
238
239             FlatSearchResponse response = indexer.searchFlat(request);
240
241             if ( response == null || response.getTotalHits() == 0 )
242             {
243                 SearchResults results = new SearchResults();
244                 results.setLimits(limits);
245                 return results;
246             }
247
248             return convertToSearchResults(response, limits, filters, principal, selectedRepos);
249         }
250         catch ( IOException e )
251         {
252             throw new RepositorySearchException(e.getMessage(), e);
253         }
254         catch ( RepositoryAdminException e )
255         {
256             throw new RepositorySearchException(e.getMessage(), e);
257         }
258
259     }
260
261     private List<IndexingContext> getIndexingContexts(List<String> ids)
262     {
263         List<IndexingContext> contexts = new ArrayList<IndexingContext>(ids.size());
264
265         for ( String id : ids )
266         {
267             IndexingContext context = indexer.getIndexingContexts().get(id);
268             if ( context != null )
269             {
270                 contexts.add(context);
271             }
272             else
273             {
274                 log.warn("context with id {} not exists", id);
275             }
276         }
277
278         return contexts;
279     }
280
281     private void constructQuery(String term, BooleanQuery q)
282     {
283         q.add(indexer.constructQuery(MAVEN.GROUP_ID, new StringSearchExpression(term)), Occur.SHOULD);
284         q.add(indexer.constructQuery(MAVEN.ARTIFACT_ID, new StringSearchExpression(term)), Occur.SHOULD);
285         q.add(indexer.constructQuery(MAVEN.VERSION, new StringSearchExpression(term)), Occur.SHOULD);
286         q.add(indexer.constructQuery(MAVEN.PACKAGING, new StringSearchExpression(term)), Occur.SHOULD);
287         q.add(indexer.constructQuery(MAVEN.CLASSNAMES, new StringSearchExpression(term)), Occur.SHOULD);
288
289         //Query query =
290         //    new WildcardQuery( new Term( MAVEN.CLASSNAMES.getFieldName(), "*" ) );
291         //q.add( query, Occur.MUST_NOT );
292         // olamy IMHO we could set this option as at least one must match
293         //q.setMinimumNumberShouldMatch( 1 );
294     }
295
296
297     /**
298      * @param selectedRepos
299      * @return indexing contextId used
300      */
301     private List<String> addIndexingContexts(List<String> selectedRepos)
302     {
303         Set<String> indexingContextIds = new HashSet<String>();
304         for ( String repo : selectedRepos )
305         {
306             try
307             {
308                 ManagedRepository repoConfig = managedRepositoryAdmin.getManagedRepository(repo);
309
310                 if ( repoConfig != null )
311                 {
312                     String indexDir = repoConfig.getIndexDirectory();
313                     File indexDirectory = null;
314                     if ( indexDir != null && !"".equals(indexDir) )
315                     {
316                         indexDirectory = new File(repoConfig.getIndexDirectory());
317                     }
318                     else
319                     {
320                         indexDirectory = new File(repoConfig.getLocation(), ".indexer");
321                     }
322
323                     IndexingContext context = indexer.getIndexingContexts().get(repoConfig.getId());
324                     if ( context != null )
325                     {
326                         // alreday here so no need to record it again
327                         log.debug("index with id {} already exists skip adding it", repoConfig.getId());
328                         // set searchable flag
329                         context.setSearchable(repoConfig.isScanned());
330                         indexingContextIds.add(context.getId());
331                         indexingContextIds.addAll(getRemoteIndexingContextIds(repo));
332                         continue;
333                     }
334
335                     context = indexer.addIndexingContext(repoConfig.getId(), repoConfig.getId(),
336                                                          new File(repoConfig.getLocation()), indexDirectory, null, null,
337                                                          getAllIndexCreators());
338                     context.setSearchable(repoConfig.isScanned());
339                     if ( context.isSearchable() )
340                     {
341                         indexingContextIds.addAll(getRemoteIndexingContextIds(repo));
342                         indexingContextIds.add(context.getId());
343                     }
344                     else
345                     {
346                         log.warn("indexingContext with id {} not searchable", repoConfig.getId());
347                     }
348
349                 }
350                 else
351                 {
352                     log.warn("Repository '" + repo + "' not found in configuration.");
353                 }
354             }
355             catch ( UnsupportedExistingLuceneIndexException e )
356             {
357                 log.warn("Error accessing index of repository '" + repo + "' : " + e.getMessage());
358                 continue;
359             }
360             catch ( IOException e )
361             {
362                 log.warn("IO error occured while accessing index of repository '" + repo + "' : " + e.getMessage());
363                 continue;
364             }
365             catch ( RepositoryAdminException e )
366             {
367                 log.warn("RepositoryAdminException occured while accessing index of repository '" + repo + "' : "
368                              + e.getMessage());
369                 continue;
370             }
371         }
372
373         return new ArrayList<String>(indexingContextIds);
374     }
375
376
377     private Set<String> getRemoteIndexingContextIds(String managedRepoId)
378         throws RepositoryAdminException
379     {
380         Set<String> ids = new HashSet<String>();
381
382         List<ProxyConnector> proxyConnectors = proxyConnectorAdmin.getProxyConnectorAsMap().get(managedRepoId);
383
384         if ( proxyConnectors == null || proxyConnectors.isEmpty() )
385         {
386             return ids;
387         }
388
389         for ( ProxyConnector proxyConnector : proxyConnectors )
390         {
391             String remoteId = "remote-" + proxyConnector.getTargetRepoId();
392             IndexingContext context = indexer.getIndexingContexts().get(remoteId);
393             if ( context != null && context.isSearchable() )
394             {
395                 ids.add(remoteId);
396             }
397         }
398
399         return ids;
400     }
401
402
403     protected List<? extends IndexCreator> getAllIndexCreators()
404     {
405         return mavenIndexerUtils.getAllIndexCreators();
406     }
407
408
409     private SearchResults convertToSearchResults(FlatSearchResponse response, SearchResultLimits limits,
410                                                  List<? extends ArtifactInfoFiler> artifactInfoFilers, String principal,
411                                                  List<String> selectedRepos)
412         throws RepositoryAdminException
413     {
414         SearchResults results = new SearchResults();
415         Set<ArtifactInfo> artifactInfos = response.getResults();
416
417         for ( ArtifactInfo artifactInfo : artifactInfos )
418         {
419             String id = SearchUtil.getHitId(artifactInfo.groupId, artifactInfo.artifactId, artifactInfo.classifier,
420                                             artifactInfo.packaging);
421             Map<String, SearchResultHit> hitsMap = results.getHitsMap();
422
423             if ( !applyArtifactInfoFilters(artifactInfo, artifactInfoFilers, hitsMap) )
424             {
425                 continue;
426             }
427
428             SearchResultHit hit = hitsMap.get(id);
429             if ( hit != null )
430             {
431                 if ( !hit.getVersions().contains(artifactInfo.version) )
432                 {
433                     hit.addVersion(artifactInfo.version);
434                 }
435             }
436             else
437             {
438                 hit = new SearchResultHit();
439                 hit.setArtifactId(artifactInfo.artifactId);
440                 hit.setGroupId(artifactInfo.groupId);
441                 hit.setRepositoryId(artifactInfo.repository);
442                 hit.addVersion(artifactInfo.version);
443                 hit.setBundleExportPackage(artifactInfo.bundleExportPackage);
444                 hit.setBundleExportService(artifactInfo.bundleExportService);
445                 hit.setBundleSymbolicName(artifactInfo.bundleSymbolicName);
446                 hit.setBundleVersion(artifactInfo.bundleVersion);
447                 hit.setBundleDescription(artifactInfo.bundleDescription);
448                 hit.setBundleDocUrl(artifactInfo.bundleDocUrl);
449                 hit.setBundleRequireBundle(artifactInfo.bundleRequireBundle);
450                 hit.setBundleImportPackage(artifactInfo.bundleImportPackage);
451                 hit.setBundleLicense(artifactInfo.bundleLicense);
452                 hit.setBundleName(artifactInfo.bundleName);
453                 hit.setContext(artifactInfo.context);
454                 hit.setGoals(artifactInfo.goals);
455                 hit.setPrefix(artifactInfo.prefix);
456                 hit.setPackaging(artifactInfo.packaging);
457                 hit.setClassifier(artifactInfo.classifier);
458                 hit.setUrl(getBaseUrl(artifactInfo, selectedRepos));
459             }
460
461             results.addHit(id, hit);
462         }
463
464         results.setTotalHits(response.getTotalHitsCount());
465         results.setReturnedHitsCount(response.getReturnedHitsCount());
466         results.setLimits(limits);
467
468         if ( limits == null || limits.getSelectedPage() == SearchResultLimits.ALL_PAGES )
469         {
470             return results;
471         }
472         else
473         {
474             return paginate(results);
475         }
476     }
477
478     /**
479      * calculate baseUrl without the context and base Archiva Url
480      *
481      * @param artifactInfo
482      * @return
483      */
484     protected String getBaseUrl(ArtifactInfo artifactInfo, List<String> selectedRepos)
485         throws RepositoryAdminException
486     {
487         StringBuilder sb = new StringBuilder();
488         if ( StringUtils.startsWith(artifactInfo.context, "remote-") )
489         {
490             // it's a remote index result we search a managed which proxying this remote and on which
491             // current user has read karma
492             String managedRepoId =
493                 getManagedRepoId(StringUtils.substringAfter(artifactInfo.context, "remote-"), selectedRepos);
494             if ( managedRepoId != null )
495             {
496                 sb.append('/').append(managedRepoId);
497             }
498         }
499         else
500         {
501             sb.append('/').append(artifactInfo.context);
502         }
503
504         sb.append('/').append(StringUtils.replaceChars(artifactInfo.groupId, '.', '/'));
505         sb.append('/').append(artifactInfo.artifactId);
506         sb.append('/').append(artifactInfo.version);
507         sb.append('/').append(artifactInfo.artifactId);
508         sb.append('-').append(artifactInfo.version);
509         if ( StringUtils.isNotBlank(artifactInfo.classifier) )
510         {
511             sb.append('-').append(artifactInfo.classifier);
512         }
513         // maven-plugin packaging is a jar
514         if ( StringUtils.equals("maven-plugin", artifactInfo.packaging) )
515         {
516             sb.append("jar");
517         }
518         else
519         {
520             sb.append('.').append(artifactInfo.packaging);
521         }
522
523         return sb.toString();
524     }
525
526     /**
527      * return a managed repo for a remote result
528      *
529      * @param remoteRepo
530      * @param selectedRepos
531      * @return
532      * @throws RepositoryAdminException
533      */
534     private String getManagedRepoId(String remoteRepo, List<String> selectedRepos)
535         throws RepositoryAdminException
536     {
537         Map<String, List<ProxyConnector>> proxyConnectorMap = proxyConnectorAdmin.getProxyConnectorAsMap();
538         if ( proxyConnectorMap == null || proxyConnectorMap.isEmpty() )
539         {
540             return null;
541         }
542         if ( selectedRepos != null && !selectedRepos.isEmpty() )
543         {
544             for ( Map.Entry<String, List<ProxyConnector>> entry : proxyConnectorMap.entrySet() )
545             {
546                 if ( selectedRepos.contains(entry.getKey()) )
547                 {
548                     for ( ProxyConnector proxyConnector : entry.getValue() )
549                     {
550                         if ( StringUtils.equals(remoteRepo, proxyConnector.getTargetRepoId()) )
551                         {
552                             return proxyConnector.getSourceRepoId();
553                         }
554                     }
555                 }
556             }
557         }
558
559         // we don't find in search selected repos so return the first one
560         for ( Map.Entry<String, List<ProxyConnector>> entry : proxyConnectorMap.entrySet() )
561         {
562
563             for ( ProxyConnector proxyConnector : entry.getValue() )
564             {
565                 if ( StringUtils.equals(remoteRepo, proxyConnector.getTargetRepoId()) )
566                 {
567                     return proxyConnector.getSourceRepoId();
568                 }
569             }
570
571         }
572         return null;
573     }
574
575     private boolean applyArtifactInfoFilters(ArtifactInfo artifactInfo,
576                                              List<? extends ArtifactInfoFiler> artifactInfoFilers,
577                                              Map<String, SearchResultHit> currentResult)
578     {
579         if ( artifactInfoFilers == null || artifactInfoFilers.isEmpty() )
580         {
581             return true;
582         }
583
584         for ( ArtifactInfoFiler filter : artifactInfoFilers )
585         {
586             if ( !filter.addArtifactInResult(artifactInfo, currentResult) )
587             {
588                 return false;
589             }
590         }
591         return true;
592     }
593
594     protected SearchResults paginate(SearchResults results)
595     {
596         SearchResultLimits limits = results.getLimits();
597         SearchResults paginated = new SearchResults();
598
599         int fetchCount = limits.getPageSize();
600         int offset = ( limits.getSelectedPage() * limits.getPageSize() );
601
602         if ( fetchCount > results.getTotalHits() )
603         {
604             fetchCount = results.getTotalHits();
605         }
606
607         // Goto offset.
608         if ( offset < results.getTotalHits() )
609         {
610             // only process if the offset is within the hit count.
611             for ( int i = 0; i < fetchCount; i++ )
612             {
613                 // Stop fetching if we are past the total # of available hits.
614                 if ( offset + i >= results.getHits().size() )
615                 {
616                     break;
617                 }
618
619                 SearchResultHit hit = results.getHits().get(( offset + i ));
620                 if ( hit != null )
621                 {
622                     String id = SearchUtil.getHitId(hit.getGroupId(), hit.getArtifactId(), hit.getClassifier(),
623                                                     hit.getPackaging());
624                     paginated.addHit(id, hit);
625                 }
626                 else
627                 {
628                     break;
629                 }
630             }
631         }
632         paginated.setTotalHits(results.getTotalHits());
633         paginated.setReturnedHitsCount(paginated.getHits().size());
634         paginated.setLimits(limits);
635
636         return paginated;
637     }
638 }