]> source.dussan.org Git - archiva.git/blob
09efd05d9b85468425b1e6d3b30e6d4e99bb7f83
[archiva.git] /
1 package org.apache.archiva.indexer.search;
2
3 /*
4  * Licensed to the Apache Software Foundation (ASF) under one
5  * or more contributor license agreements.  See the NOTICE file
6  * distributed with this work for additional information
7  * regarding copyright ownership.  The ASF licenses this file
8  * to you under the Apache License, Version 2.0 (the
9  * "License"); you may not use this file except in compliance
10  * with the License.  You may obtain a copy of the License at
11  *
12  *  http://www.apache.org/licenses/LICENSE-2.0
13  *
14  * Unless required by applicable law or agreed to in writing,
15  * software distributed under the License is distributed on an
16  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17  * KIND, either express or implied.  See the License for the
18  * specific language governing permissions and limitations
19  * under the License.
20  */
21
22 import org.apache.archiva.admin.model.RepositoryAdminException;
23 import org.apache.archiva.admin.model.beans.ManagedRepository;
24 import org.apache.archiva.admin.model.beans.ProxyConnector;
25 import org.apache.archiva.admin.model.managed.ManagedRepositoryAdmin;
26 import org.apache.archiva.admin.model.proxyconnector.ProxyConnectorAdmin;
27 import org.apache.archiva.common.plexusbridge.MavenIndexerUtils;
28 import org.apache.archiva.common.plexusbridge.PlexusSisuBridge;
29 import org.apache.archiva.common.plexusbridge.PlexusSisuBridgeException;
30 import org.apache.archiva.indexer.util.SearchUtil;
31 import org.apache.commons.lang.StringUtils;
32 import org.apache.lucene.search.BooleanClause.Occur;
33 import org.apache.lucene.search.BooleanQuery;
34 import org.apache.maven.index.ArtifactInfo;
35 import org.apache.maven.index.FlatSearchRequest;
36 import org.apache.maven.index.FlatSearchResponse;
37 import org.apache.maven.index.MAVEN;
38 import org.apache.maven.index.NexusIndexer;
39 import org.apache.maven.index.OSGI;
40 import org.apache.maven.index.context.IndexCreator;
41 import org.apache.maven.index.context.IndexingContext;
42 import org.apache.maven.index.context.UnsupportedExistingLuceneIndexException;
43 import org.apache.maven.index.expr.StringSearchExpression;
44 import org.slf4j.Logger;
45 import org.slf4j.LoggerFactory;
46 import org.springframework.stereotype.Service;
47
48 import javax.inject.Inject;
49 import java.io.File;
50 import java.io.IOException;
51 import java.util.ArrayList;
52 import java.util.Collections;
53 import java.util.HashSet;
54 import java.util.List;
55 import java.util.Map;
56 import java.util.Set;
57
58 /**
59  * RepositorySearch implementation which uses the Nexus Indexer for searching.
60  */
61 @Service( "nexusSearch" )
62 public class NexusRepositorySearch
63     implements RepositorySearch
64 {
65     private Logger log = LoggerFactory.getLogger( getClass() );
66
67     private NexusIndexer indexer;
68
69     private ManagedRepositoryAdmin managedRepositoryAdmin;
70
71     private ProxyConnectorAdmin proxyConnectorAdmin;
72
73     private MavenIndexerUtils mavenIndexerUtils;
74
75     @Inject
76     public NexusRepositorySearch( PlexusSisuBridge plexusSisuBridge, ManagedRepositoryAdmin managedRepositoryAdmin,
77                                   MavenIndexerUtils mavenIndexerUtils, ProxyConnectorAdmin proxyConnectorAdmin )
78         throws PlexusSisuBridgeException
79     {
80         this.indexer = plexusSisuBridge.lookup( NexusIndexer.class );
81         this.managedRepositoryAdmin = managedRepositoryAdmin;
82         this.mavenIndexerUtils = mavenIndexerUtils;
83         this.proxyConnectorAdmin = proxyConnectorAdmin;
84     }
85
86     /**
87      * @see RepositorySearch#search(String, List, String, SearchResultLimits, List)
88      */
89     public SearchResults search( String principal, List<String> selectedRepos, String term, SearchResultLimits limits,
90                                  List<String> previousSearchTerms )
91         throws RepositorySearchException
92     {
93         List<String> indexingContextIds = addIndexingContexts( selectedRepos );
94
95         // since upgrade to nexus 2.0.0, query has changed from g:[QUERIED TERM]* to g:*[QUERIED TERM]*
96         //      resulting to more wildcard searches so we need to increase max clause count
97         BooleanQuery.setMaxClauseCount( Integer.MAX_VALUE );
98         BooleanQuery q = new BooleanQuery();
99
100         if ( previousSearchTerms == null || previousSearchTerms.isEmpty() )
101         {
102             constructQuery( term, q );
103         }
104         else
105         {
106             for ( String previousTerm : previousSearchTerms )
107             {
108                 BooleanQuery iQuery = new BooleanQuery();
109                 constructQuery( previousTerm, iQuery );
110
111                 q.add( iQuery, Occur.MUST );
112             }
113
114             BooleanQuery iQuery = new BooleanQuery();
115             constructQuery( term, iQuery );
116             q.add( iQuery, Occur.MUST );
117         }
118
119         // we retun only artifacts without classifier in quick search, olamy cannot find a way to say with this field empty
120         // FIXME  cannot find a way currently to setup this in constructQuery !!!
121         return search( limits, q, indexingContextIds, NoClassifierArtifactInfoFiler.LIST, principal, selectedRepos );
122
123     }
124
125     /**
126      * @see RepositorySearch#search(String, SearchFields, SearchResultLimits)
127      */
128     public SearchResults search( String principal, SearchFields searchFields, SearchResultLimits limits )
129         throws RepositorySearchException
130     {
131         if ( searchFields.getRepositories() == null )
132         {
133             throw new RepositorySearchException( "Repositories cannot be null." );
134         }
135
136         List<String> indexingContextIds = addIndexingContexts( searchFields.getRepositories() );
137
138         BooleanQuery q = new BooleanQuery();
139         if ( StringUtils.isNotBlank( searchFields.getGroupId() ) )
140         {
141             q.add( indexer.constructQuery( MAVEN.GROUP_ID, new StringSearchExpression( searchFields.getGroupId() ) ),
142                    Occur.MUST );
143         }
144
145         if ( StringUtils.isNotBlank( searchFields.getArtifactId() ) )
146         {
147             q.add(
148                 indexer.constructQuery( MAVEN.ARTIFACT_ID, new StringSearchExpression( searchFields.getArtifactId() ) ),
149                 Occur.MUST );
150         }
151
152         if ( StringUtils.isNotBlank( searchFields.getVersion() ) )
153         {
154             q.add( indexer.constructQuery( MAVEN.VERSION, new StringSearchExpression( searchFields.getVersion() ) ),
155                    Occur.MUST );
156         }
157
158         if ( StringUtils.isNotBlank( searchFields.getPackaging() ) )
159         {
160             q.add( indexer.constructQuery( MAVEN.PACKAGING, new StringSearchExpression( searchFields.getPackaging() ) ),
161                    Occur.MUST );
162         }
163
164         if ( StringUtils.isNotBlank( searchFields.getClassName() ) )
165         {
166             q.add(
167                 indexer.constructQuery( MAVEN.CLASSNAMES, new StringSearchExpression( searchFields.getClassName() ) ),
168                 Occur.MUST );
169         }
170
171         if ( StringUtils.isNotBlank( searchFields.getBundleSymbolicName() ) )
172         {
173             q.add( indexer.constructQuery( OSGI.SYMBOLIC_NAME,
174                                            new StringSearchExpression( searchFields.getBundleSymbolicName() ) ),
175                    Occur.MUST );
176         }
177
178         if ( StringUtils.isNotBlank( searchFields.getBundleVersion() ) )
179         {
180             q.add(
181                 indexer.constructQuery( OSGI.VERSION, new StringSearchExpression( searchFields.getBundleVersion() ) ),
182                 Occur.MUST );
183         }
184
185         if ( StringUtils.isNotBlank( searchFields.getBundleExportPackage() ) )
186         {
187             q.add( indexer.constructQuery( OSGI.EXPORT_PACKAGE,
188                                            new StringSearchExpression( searchFields.getBundleExportPackage() ) ),
189                    Occur.MUST );
190         }
191
192         if ( StringUtils.isNotBlank( searchFields.getBundleExportService() ) )
193         {
194             q.add( indexer.constructQuery( OSGI.EXPORT_SERVICE,
195                                            new StringSearchExpression( searchFields.getBundleExportService() ) ),
196                    Occur.MUST );
197         }
198
199         if ( StringUtils.isNotBlank( searchFields.getBundleImportPackage() ) )
200         {
201             q.add( indexer.constructQuery( OSGI.IMPORT_PACKAGE,
202                                            new StringSearchExpression( searchFields.getBundleImportPackage() ) ),
203                    Occur.MUST );
204         }
205
206         if ( StringUtils.isNotBlank( searchFields.getBundleName() ) )
207         {
208             q.add( indexer.constructQuery( OSGI.NAME, new StringSearchExpression( searchFields.getBundleName() ) ),
209                    Occur.MUST );
210         }
211
212         if ( StringUtils.isNotBlank( searchFields.getClassifier() ) )
213         {
214             q.add(
215                 indexer.constructQuery( MAVEN.CLASSIFIER, new StringSearchExpression( searchFields.getClassifier() ) ),
216                 Occur.MUST );
217         }
218
219         if ( q.getClauses() == null || q.getClauses().length <= 0 )
220         {
221             throw new RepositorySearchException( "No search fields set." );
222         }
223
224         return search( limits, q, indexingContextIds, Collections.<ArtifactInfoFiler>emptyList(), principal,
225                        searchFields.getRepositories() );
226     }
227
228     private SearchResults search( SearchResultLimits limits, BooleanQuery q, List<String> indexingContextIds,
229                                   List<? extends ArtifactInfoFiler> filters, String principal,
230                                   List<String> selectedRepos )
231         throws RepositorySearchException
232     {
233
234         try
235         {
236             FlatSearchRequest request = new FlatSearchRequest( q );
237             request.setContexts( getIndexingContexts( indexingContextIds ) );
238             FlatSearchResponse response = indexer.searchFlat( request );
239
240             if ( response == null || response.getTotalHits() == 0 )
241             {
242                 SearchResults results = new SearchResults();
243                 results.setLimits( limits );
244                 return results;
245             }
246
247             return convertToSearchResults( response, limits, filters, principal, selectedRepos );
248         }
249         catch ( IOException e )
250         {
251             throw new RepositorySearchException( e.getMessage(), e );
252         }
253         catch ( RepositoryAdminException e )
254         {
255             throw new RepositorySearchException( e.getMessage(), e );
256         }
257
258     }
259
260     private List<IndexingContext> getIndexingContexts( List<String> ids )
261     {
262         List<IndexingContext> contexts = new ArrayList<IndexingContext>( ids.size() );
263
264         for ( String id : ids )
265         {
266             IndexingContext context = indexer.getIndexingContexts().get( id );
267             if ( context != null )
268             {
269                 contexts.add( context );
270             }
271             else
272             {
273                 log.warn( "context with id {} not exists", id );
274             }
275         }
276
277         return contexts;
278     }
279
280     private void constructQuery( String term, BooleanQuery q )
281     {
282         q.add( indexer.constructQuery( MAVEN.GROUP_ID, new StringSearchExpression( term ) ), Occur.SHOULD );
283         q.add( indexer.constructQuery( MAVEN.ARTIFACT_ID, new StringSearchExpression( term ) ), Occur.SHOULD );
284         q.add( indexer.constructQuery( MAVEN.VERSION, new StringSearchExpression( term ) ), Occur.SHOULD );
285         q.add( indexer.constructQuery( MAVEN.PACKAGING, new StringSearchExpression( term ) ), Occur.SHOULD );
286         q.add( indexer.constructQuery( MAVEN.CLASSNAMES, new StringSearchExpression( term ) ), Occur.SHOULD );
287
288         //Query query =
289         //    new WildcardQuery( new Term( MAVEN.CLASSNAMES.getFieldName(), "*" ) );
290         //q.add( query, Occur.MUST_NOT );
291         // olamy IMHO we could set this option as at least one must match
292         //q.setMinimumNumberShouldMatch( 1 );
293     }
294
295
296     /**
297      * @param selectedRepos
298      * @return indexing contextId used
299      */
300     private List<String> addIndexingContexts( List<String> selectedRepos )
301     {
302         Set<String> indexingContextIds = new HashSet<String>();
303         for ( String repo : selectedRepos )
304         {
305             try
306             {
307                 ManagedRepository repoConfig = managedRepositoryAdmin.getManagedRepository( repo );
308
309                 if ( repoConfig != null )
310                 {
311                     String indexDir = repoConfig.getIndexDirectory();
312                     File indexDirectory = null;
313                     if ( indexDir != null && !"".equals( indexDir ) )
314                     {
315                         indexDirectory = new File( repoConfig.getIndexDirectory() );
316                     }
317                     else
318                     {
319                         indexDirectory = new File( repoConfig.getLocation(), ".indexer" );
320                     }
321
322                     IndexingContext context = indexer.getIndexingContexts().get( repoConfig.getId() );
323                     if ( context != null )
324                     {
325                         // alreday here so no need to record it again
326                         log.debug( "index with id {} already exists skip adding it", repoConfig.getId() );
327                         // set searchable flag
328                         context.setSearchable( repoConfig.isScanned() );
329                         indexingContextIds.add( context.getId() );
330                         indexingContextIds.addAll( getRemoteIndexingContextIds( repo ) );
331                         continue;
332                     }
333
334                     context = indexer.addIndexingContext( repoConfig.getId(), repoConfig.getId(),
335                                                           new File( repoConfig.getLocation() ), indexDirectory, null,
336                                                           null, getAllIndexCreators() );
337                     context.setSearchable( repoConfig.isScanned() );
338                     if ( context.isSearchable() )
339                     {
340                         indexingContextIds.addAll( getRemoteIndexingContextIds( repo ) );
341                         indexingContextIds.add( context.getId() );
342                     }
343                     else
344                     {
345                         log.warn( "indexingContext with id {} not searchable", repoConfig.getId() );
346                     }
347
348                 }
349                 else
350                 {
351                     log.warn( "Repository '" + repo + "' not found in configuration." );
352                 }
353             }
354             catch ( UnsupportedExistingLuceneIndexException e )
355             {
356                 log.warn( "Error accessing index of repository '" + repo + "' : " + e.getMessage() );
357                 continue;
358             }
359             catch ( IOException e )
360             {
361                 log.warn( "IO error occured while accessing index of repository '" + repo + "' : " + e.getMessage() );
362                 continue;
363             }
364             catch ( RepositoryAdminException e )
365             {
366                 log.warn( "RepositoryAdminException occured while accessing index of repository '" + repo + "' : "
367                               + e.getMessage() );
368                 continue;
369             }
370         }
371
372         return new ArrayList<String>( indexingContextIds );
373     }
374
375
376     private Set<String> getRemoteIndexingContextIds( String managedRepoId )
377         throws RepositoryAdminException
378     {
379         Set<String> ids = new HashSet<String>();
380
381         List<ProxyConnector> proxyConnectors = proxyConnectorAdmin.getProxyConnectorAsMap().get( managedRepoId );
382
383         if ( proxyConnectors == null || proxyConnectors.isEmpty() )
384         {
385             return ids;
386         }
387
388         for ( ProxyConnector proxyConnector : proxyConnectors )
389         {
390             String remoteId = "remote-" + proxyConnector.getTargetRepoId();
391             IndexingContext context = indexer.getIndexingContexts().get( remoteId );
392             if ( context != null && context.isSearchable() )
393             {
394                 ids.add( remoteId );
395             }
396         }
397
398         return ids;
399     }
400
401
402     protected List<? extends IndexCreator> getAllIndexCreators()
403     {
404         return mavenIndexerUtils.getAllIndexCreators();
405     }
406
407
408     private SearchResults convertToSearchResults( FlatSearchResponse response, SearchResultLimits limits,
409                                                   List<? extends ArtifactInfoFiler> artifactInfoFilers,
410                                                   String principal, List<String> selectedRepos )
411         throws RepositoryAdminException
412     {
413         SearchResults results = new SearchResults();
414         Set<ArtifactInfo> artifactInfos = response.getResults();
415
416         for ( ArtifactInfo artifactInfo : artifactInfos )
417         {
418             String id = SearchUtil.getHitId( artifactInfo.groupId, artifactInfo.artifactId, artifactInfo.classifier,
419                                              artifactInfo.packaging );
420             Map<String, SearchResultHit> hitsMap = results.getHitsMap();
421
422             if ( !applyArtifactInfoFilters( artifactInfo, artifactInfoFilers, hitsMap ) )
423             {
424                 continue;
425             }
426
427             SearchResultHit hit = hitsMap.get( id );
428             if ( hit != null )
429             {
430                 if ( !hit.getVersions().contains( artifactInfo.version ) )
431                 {
432                     hit.addVersion( artifactInfo.version );
433                 }
434             }
435             else
436             {
437                 hit = new SearchResultHit();
438                 hit.setArtifactId( artifactInfo.artifactId );
439                 hit.setGroupId( artifactInfo.groupId );
440                 hit.setRepositoryId( artifactInfo.repository );
441                 hit.addVersion( artifactInfo.version );
442                 hit.setBundleExportPackage( artifactInfo.bundleExportPackage );
443                 hit.setBundleExportService( artifactInfo.bundleExportService );
444                 hit.setBundleSymbolicName( artifactInfo.bundleSymbolicName );
445                 hit.setBundleVersion( artifactInfo.bundleVersion );
446                 hit.setBundleDescription( artifactInfo.bundleDescription );
447                 hit.setBundleDocUrl( artifactInfo.bundleDocUrl );
448                 hit.setBundleRequireBundle( artifactInfo.bundleRequireBundle );
449                 hit.setBundleImportPackage( artifactInfo.bundleImportPackage );
450                 hit.setBundleLicense( artifactInfo.bundleLicense );
451                 hit.setBundleName( artifactInfo.bundleName );
452                 hit.setContext( artifactInfo.context );
453                 hit.setGoals( artifactInfo.goals );
454                 hit.setPrefix( artifactInfo.prefix );
455                 hit.setPackaging( artifactInfo.packaging );
456                 hit.setClassifier( artifactInfo.classifier );
457                 hit.setUrl( getBaseUrl( artifactInfo, selectedRepos ) );
458             }
459
460             results.addHit( id, hit );
461         }
462
463         results.setTotalHits( response.getTotalHitsCount() );
464         results.setReturnedHitsCount( response.getReturnedHitsCount() );
465         results.setLimits( limits );
466
467         if ( limits == null || limits.getSelectedPage() == SearchResultLimits.ALL_PAGES )
468         {
469             return results;
470         }
471         else
472         {
473             return paginate( results );
474         }
475     }
476
477     /**
478      * calculate baseUrl without the context and base Archiva Url
479      *
480      * @param artifactInfo
481      * @return
482      */
483     protected String getBaseUrl( ArtifactInfo artifactInfo, List<String> selectedRepos )
484         throws RepositoryAdminException
485     {
486         StringBuilder sb = new StringBuilder();
487         if ( StringUtils.startsWith( artifactInfo.context, "remote-" ) )
488         {
489             // it's a remote index result we search a managed which proxying this remote and on which
490             // current user has read karma
491             String managedRepoId =
492                 getManagedRepoId( StringUtils.substringAfter( artifactInfo.context, "remote-" ), selectedRepos );
493             if ( managedRepoId != null )
494             {
495                 sb.append( '/' ).append( managedRepoId );
496             }
497         }
498         else
499         {
500             sb.append( '/' ).append( artifactInfo.context );
501         }
502
503         sb.append( '/' ).append( StringUtils.replaceChars( artifactInfo.groupId, '.', '/' ) );
504         sb.append( '/' ).append( artifactInfo.artifactId );
505         sb.append( '/' ).append( artifactInfo.version );
506         sb.append( '/' ).append( artifactInfo.artifactId );
507         sb.append( '-' ).append( artifactInfo.version );
508         if ( StringUtils.isNotBlank( artifactInfo.classifier ) )
509         {
510             sb.append( '-' ).append( artifactInfo.classifier );
511         }
512         // maven-plugin packaging is a jar
513         if ( StringUtils.equals( "maven-plugin", artifactInfo.packaging ) )
514         {
515             sb.append( "jar" );
516         }
517         else
518         {
519             sb.append( '.' ).append( artifactInfo.packaging );
520         }
521
522         return sb.toString();
523     }
524
525     /**
526      * return a managed repo for a remote result
527      *
528      * @param remoteRepo
529      * @param selectedRepos
530      * @return
531      * @throws RepositoryAdminException
532      */
533     private String getManagedRepoId( String remoteRepo, List<String> selectedRepos )
534         throws RepositoryAdminException
535     {
536         Map<String, List<ProxyConnector>> proxyConnectorMap = proxyConnectorAdmin.getProxyConnectorAsMap();
537         if ( proxyConnectorMap == null || proxyConnectorMap.isEmpty() )
538         {
539             return null;
540         }
541         if ( selectedRepos != null && !selectedRepos.isEmpty() )
542         {
543             for ( Map.Entry<String, List<ProxyConnector>> entry : proxyConnectorMap.entrySet() )
544             {
545                 if ( selectedRepos.contains( entry.getKey() ) )
546                 {
547                     for ( ProxyConnector proxyConnector : entry.getValue() )
548                     {
549                         if ( StringUtils.equals( remoteRepo, proxyConnector.getTargetRepoId() ) )
550                         {
551                             return proxyConnector.getSourceRepoId();
552                         }
553                     }
554                 }
555             }
556         }
557
558         // we don't find in search selected repos so return the first one
559         for ( Map.Entry<String, List<ProxyConnector>> entry : proxyConnectorMap.entrySet() )
560         {
561
562             for ( ProxyConnector proxyConnector : entry.getValue() )
563             {
564                 if ( StringUtils.equals( remoteRepo, proxyConnector.getTargetRepoId() ) )
565                 {
566                     return proxyConnector.getSourceRepoId();
567                 }
568             }
569
570         }
571         return null;
572     }
573
574     private boolean applyArtifactInfoFilters( ArtifactInfo artifactInfo,
575                                               List<? extends ArtifactInfoFiler> artifactInfoFilers,
576                                               Map<String, SearchResultHit> currentResult )
577     {
578         if ( artifactInfoFilers == null || artifactInfoFilers.isEmpty() )
579         {
580             return true;
581         }
582
583         for ( ArtifactInfoFiler filter : artifactInfoFilers )
584         {
585             if ( !filter.addArtifactInResult( artifactInfo, currentResult ) )
586             {
587                 return false;
588             }
589         }
590         return true;
591     }
592
593     private SearchResults paginate( SearchResults results )
594     {
595         SearchResultLimits limits = results.getLimits();
596         SearchResults paginated = new SearchResults();
597
598         int fetchCount = limits.getPageSize();
599         int offset = ( limits.getSelectedPage() * limits.getPageSize() );
600
601         if ( fetchCount > results.getTotalHits() )
602         {
603             fetchCount = results.getTotalHits();
604         }
605
606         // Goto offset.
607         if ( offset < results.getTotalHits() )
608         {
609             // only process if the offset is within the hit count.
610             for ( int i = 0; i < fetchCount; i++ )
611             {
612                 // Stop fetching if we are past the total # of available hits.
613                 if ( offset + i >= results.getHits().size() )
614                 {
615                     break;
616                 }
617
618                 SearchResultHit hit = results.getHits().get( ( offset + i ) );
619                 if ( hit != null )
620                 {
621                     String id = SearchUtil.getHitId( hit.getGroupId(), hit.getArtifactId(), hit.getClassifier(),
622                                                      hit.getPackaging() );
623                     paginated.addHit( id, hit );
624                 }
625                 else
626                 {
627                     break;
628                 }
629             }
630         }
631         paginated.setTotalHits( results.getTotalHits() );
632         paginated.setReturnedHitsCount( paginated.getHits().size() );
633         paginated.setLimits( limits );
634
635         return paginated;
636     }
637 }